btreejs (#35)

* btree -- rough draft

* add compareBytes test

* resolve comments, test out bsearch + comparebytes

src/btree.ts

@@ -0,0 +1,124 @@
+import { BPTreeNode, MemoryPointer, compareBytes } from "./node";
+import { LengthIntegrityError, RangeResolver } from "./resolver";
+
+// taken from `buffer.go`
+interface MetaPage {
+	root(): Promise<MemoryPointer>;
+}
+
+class BPTree {
+	private tree: RangeResolver;
+	private meta: MetaPage;
+	private maxPageSize: number;
+
+	constructor(tree: RangeResolver, meta: MetaPage, maxPageSize: number) {
+		this.tree = tree;
+		this.meta = meta;
+		this.maxPageSize = maxPageSize;
+	}
+
+	private async root(): Promise<[BPTreeNode | null, MemoryPointer]> {
+		const mp = await this.meta.root();
+		if (!mp || mp.length === 0) {
+			return [null, mp];
+		}
+
+		const root = await this.readNode(mp);
+		if (!root) {
+			return [null, mp];
+		}
+
+		return [root, mp];
+	}
+
+	private async readNode(ptr: MemoryPointer): Promise<BPTreeNode | null> {
+		try {
+			const node = new BPTreeNode(this.tree, [], []);
+
+			const bytesRead = await node.readFrom();
+
+			if (!bytesRead || bytesRead !== ptr.length) {
+				return null;
+			}
+
+			return node;
+		} catch (error) {
+			if (error instanceof LengthIntegrityError) {
+				// handle LengthIntegrityError
+			}
+
+			return null;
+		}
+	}
+
+	private async traverse(
+		key: Uint8Array,
+		node: BPTreeNode
+	): Promise<TraversalRecord[] | null> {
+		if (await node.leaf()) {
+			return [{ node: node, index: 0 }];
+		}
+
+		for (const [i, k] of node.keys.entries()) {
+			if (compareBytes(key, k.value) < 0) {
+				const child = await this.readNode(node.pointers[i]);
+				if (!child) {
+					return null;
+				}
+
+				const path = await this.traverse(key, child);
+				if (!path) {
+					return null;
+				}
+
+				return [...path, { node: node, index: i }];
+			}
+		}
+
+		const child = await this.readNode(node.pointers[node.pointers.length - 1]);
+
+		if (!child) {
+			return null;
+		}
+
+		const path = await this.traverse(key, child);
+		if (!path) {
+			return null;
+		}
+
+		return [...path, { node: node, index: node.keys.length }];
+	}
+
+	public async find(key: Uint8Array): Promise<[MemoryPointer, boolean]> {
+		let [rootNode, _] = await this.root();
+
+		if (!rootNode) {
+			return [{ offset: 0, length: 0 }, false];
+		}
+
+		const path = await this.traverse(key, rootNode);
+		if (!path) {
+			return [{ offset: 0, length: 0 }, false];
+		}
+
+		const n = path[0].node;
+
+		const i = await n.bsearch(key);
+
+		if (i >= 0) {
+			return [n.pointers[i], true];
+		}
+
+		return [{ offset: 0, length: 0 }, false];
+	}
+}
+
+class TraversalRecord {
+	public node: BPTreeNode;
+	public index: number;
+
+	constructor(node: BPTreeNode, index: number) {
+		this.node = node;
+		this.index = index;
+	}
+}

src/database.ts

@@ -184,11 +184,8 @@ export class Database<T extends Schema> {
 		// evaluate the field ranges in order.
 		for (const [key, [start, end]] of fieldRangesSorted) {
 			// check if the iteration order should be reversed.
-			console.log(query.orderBy);
 			const orderBy = query.orderBy?.find((orderBy) => orderBy.key === key);
-			console.log(orderBy);
 			const reverse = orderBy?.direction === "DESC";
-			console.log(key, start, end, reverse);
 			const length = end - start;
 			for (let offset = 0; offset < length; offset++) {
 				const index = reverse ? end - offset - 1 : start + offset;

src/node.ts

@@ -0,0 +1,190 @@
+import { RangeResolver } from "./resolver";
+
+export type ReferencedValue = { dataPointer: MemoryPointer; value: Buffer };
+export type MemoryPointer = { offset: number; length: number };
+
+export class BPTreeNode {
+	public dataHandler: RangeResolver;
+	public pointers: MemoryPointer[];
+	public keys: ReferencedValue[];
+
+	constructor(
+		dataHandler: RangeResolver,
+		pointers: MemoryPointer[],
+		keys: ReferencedValue[]
+	) {
+		this.dataHandler = dataHandler;
+		this.pointers = pointers;
+		this.keys = keys;
+	}
+
+	leaf(): boolean {
+		return this.pointers.length === this.keys.length;
+	}
+
+	async readFrom(): Promise<number> {
+		let totalBytesRead = 0;
+
+		try {
+			console.log("Fetching initial data...");
+
+			let { data: sizeData } = await this.dataHandler({
+				start: 0,
+				end: 4,
+			});
+
+			let sizeBuffer = Buffer.from(sizeData);
+
+			let size = sizeBuffer.readInt32BE(0);
+			let leaf = size < 0;
+			let absSize = Math.abs(size);
+
+			console.log(`Size: ${size}, Leaf: ${leaf}`);
+
+			this.pointers = new Array(absSize + (leaf ? 0 : 1))
+				.fill(null)
+				.map(() => ({ offset: 0, length: 0 }));
+			this.keys = new Array(absSize).fill(null).map(() => ({
+				dataPointer: { offset: 0, length: 0 },
+				value: Buffer.alloc(0),
+			}));
+
+			let currentOffset = 4;
+			totalBytesRead += 4;
+
+			for (let idx = 0; idx <= this.keys.length - 1; idx++) {
+				console.log(`Processing key ${idx}...`);
+
+				let { data: keyData } = await this.dataHandler({
+					start: currentOffset,
+					end: currentOffset + 4,
+				});
+
+				console.log(`Key data fetched:`, keyData);
+
+				let keyBuffer = Buffer.from(keyData);
+				let l = keyBuffer.readUint32BE(0);
+
+				console.log("length of key", l);
+
+				currentOffset += 4;
+				totalBytesRead += 4;
+
+				if (l === 0) {
+					let { data: pointerData } = await this.dataHandler({
+						start: currentOffset,
+						end: currentOffset + 12,
+					});
+					let pointerBuffer = Buffer.from(pointerData);
+
+					let dpOffset = pointerBuffer.readInt32BE(0);
+					let dpLength = pointerBuffer.readUInt32BE(4);
+
+					this.keys[idx].dataPointer = { offset: dpOffset, length: dpLength };
+					currentOffset += 8;
+					totalBytesRead += 8;
+
+					let { data: keyValue } = await this.dataHandler({
+						start: dpOffset,
+						end: dpOffset + dpLength - 1,
+					});
+					this.keys[idx].value = Buffer.from(keyValue);
+					this.keys[idx].dataPointer.length = dpLength;
+
+					totalBytesRead += dpLength;
+				} else {
+					let { data: keyValue } = await this.dataHandler({
+						start: currentOffset,
+						end: currentOffset + l,
+					});
+
+					console.log(
+						"key value from buffer: ",
+						Buffer.from(keyValue).toString()
+					);
+					this.keys[idx].value = Buffer.from(keyValue);
+					this.keys[idx].dataPointer.length = l; // directly assign length here
+
+					currentOffset += l;
+					totalBytesRead += l;
+				}
+			}
+
+			for (let idx = 0; idx <= this.pointers.length - 1; idx++) {
+				console.log("reading from currentOffset: ", currentOffset);
+
+				let { data: offsetData } = await this.dataHandler({
+					start: currentOffset,
+					end: currentOffset + 4,
+				});
+				let offsetBuffer = Buffer.from(offsetData);
+
+				let pointerOffset = offsetBuffer.readUint32BE(0);
+				currentOffset += 4;
+				totalBytesRead += 4;
+
+				console.log("reading from currentOffset: ", currentOffset);
+				let { data: lengthData } = await this.dataHandler({
+					start: currentOffset,
+					end: currentOffset + 4,
+				});
+				let lengthBuffer = Buffer.from(lengthData);
+
+				let pointerLength = lengthBuffer.readUint32BE(0);
+				currentOffset += 4;
+				totalBytesRead += 4;
+
+				this.pointers[idx] = { offset: pointerOffset, length: pointerLength };
+
+				totalBytesRead += 8;
+			}
+		} catch (error) {
+			// console.error(error);
+			return 0;
+		}
+
+		return totalBytesRead;
+	}
+
+	async bsearch(key: Uint8Array): Promise<number> {
+		let lo = 0;
+		let hi = this.keys.length - 1;
+
+		while (lo <= hi) {
+			const mid = Math.floor((lo + hi) / 2);
+			const cmp = compareBytes(key, this.keys[mid].value);
+
+			switch (cmp) {
+				case 0:
+					return mid;
+				case -1:
+					hi = mid - 1;
+					break;
+				case 1:
+					lo = mid + 1;
+					break;
+			}
+		}
+
+		return ~lo;
+	}
+}
+
+export function compareBytes(a: Uint8Array, b: Uint8Array): number {
+	const len = Math.min(a.length, b.length);
+
+	for (let idx = 0; idx < len; idx++) {
+		if (a[idx] !== b[idx]) {
+			return a[idx] < b[idx] ? -1 : 1;
+		}
+	}
+
+	if (a.length < b.length) {
+		return -1;
+	}
+	if (a.length > b.length) {
+		return 1;
+	}
+
+	return 0;
+}

src/tests/database.test.ts

@@ -5,7 +5,7 @@ import { IndexFile, VersionedIndexFile } from "../index-file";
 jest.mock("../data-file");
 jest.mock("../index-file");
 
-describe("Database", () => {
+describe("test query relation", () => {
 	let mockDataFile: jest.Mocked<DataFile>;
 	let mockIndexFile: jest.Mocked<VersionedIndexFile<any>>;
 	let database: Database<any>;
@@ -24,12 +24,11 @@ describe("Database", () => {
 			dataRecord: jest.fn(),
 		} as jest.Mocked<VersionedIndexFile<any>>;
 
-
-        // instantiate a Database object with given mocked data file and index file
+		// instantiate a Database object with given mocked data file and index file
 		database = Database.forDataFileAndIndexFile(mockDataFile, mockIndexFile);
 	});
 
-    /*
+	/*
     This test case tests the query function in `database.ts`.
     */
 	it("should handle a simple query", async () => {
@@ -52,10 +51,10 @@ describe("Database", () => {
 			fieldLength: 10,
 		});
 
-        mockIndexFile.dataRecord.mockResolvedValue({
-            startByteOffset: 0,
-            endByteOffset: 10,
-        });
+		mockIndexFile.dataRecord.mockResolvedValue({
+			startByteOffset: 0,
+			endByteOffset: 10,
+		});
 
 		// Adjust the mocked DataFile.get to return a string that represents a valid JSON object
 		mockDataFile.get.mockImplementation(