more work

pkg/btree/node_test.go

@@ -6,6 +6,15 @@ import (
 	"testing"
 )
 
+/*
+func writeBufferToFile(buf *bytes.Buffer, filename string) error {
+	if err := os.WriteFile(filename, buf.Bytes(), 0644); err != nil {
+		return err
+	}
+	return nil
+}
+*/
+
 func TestBPTreeNode_ReadWriteLeaf(t *testing.T) {
 	// Create a test BPTreeNode
 	node1 := &BPTreeNode{

pkg/btree/pagefile.go

@@ -27,7 +27,7 @@ type PageFile struct {
 
 var _ ReadWriteSeekPager = &PageFile{}
 
-const maxFreePageIndices = 512
+// const maxFreePageIndices = 512
 const pageSizeBytes = 4096 // 4kB by default.
 
 func NewPageFile(rws io.ReadWriteSeeker) (*PageFile, error) {

src/btree/bptree.ts

@@ -16,9 +16,9 @@ type RootResponse = {
 };
 
 export class BPTree {
-	private tree: RangeResolver;
+	private readonly tree: RangeResolver;
 	private meta: MetaPage;
-	private data: Uint8Array;
+	private readonly data: Uint8Array;
 
 	constructor(tree: RangeResolver, meta: MetaPage, data: Uint8Array) {
 		this.tree = tree;
@@ -62,7 +62,7 @@ export class BPTree {
 	}
 
 	private async traverse(
-		key: Uint8Array,
+		key: ArrayBuffer,
 		node: BPTreeNode,
 		pointer: MemoryPointer
 	): Promise<TraversalRecord[]> {
@@ -121,14 +121,17 @@ class TraversalRecord {
 
 export class ReferencedValue {
 	public dataPointer: MemoryPointer;
-	public value: Uint8Array;
+	public value: ArrayBuffer;
 
 	constructor(dataPointer: MemoryPointer, value: Uint8Array) {
 		this.dataPointer = dataPointer;
 		this.value = value;
 	}
 
-	static compareBytes(a: Uint8Array, b: Uint8Array): number {
+	static compareBytes(aBuffer: ArrayBuffer, bBuffer: ArrayBuffer): number {
+		const a = new Uint8Array(aBuffer);
+		const b = new Uint8Array(bBuffer);
+
 		const len = Math.min(a.length, b.length);
 		for (let idx = 0; idx < len; idx++) {
 			if (a[idx] !== b[idx]) {
@@ -148,7 +151,7 @@ export class ReferencedValue {
 
 function binarySearchReferencedValues(
 	values: ReferencedValue[],
-	target: Uint8Array
+	target: ArrayBuffer
 ): number {
 	let lo = 0;
 	let hi = values.length;

src/btree/multi.ts

@@ -1,25 +1,25 @@
 import { RangeResolver } from "../resolver";
-import { BPTree } from "./bptree";
 import { MemoryPointer } from "./node";
+import {PageFile} from "./pagefile";
 
 const PAGE_SIZE_BYTES = 4096;
 
 export class LinkedMetaPage {
 	private resolver: RangeResolver;
-	private offset: number;
-	private pageData: ArrayBuffer | null;
+	private offset: bigint;
+	private metaPageData: ArrayBuffer | null;
 
-	constructor(resolver: RangeResolver, offset: number) {
+	constructor(resolver: RangeResolver, offset: bigint) {
 		this.resolver = resolver;
 		this.offset = offset;
-		this.pageData = null;
+		this.metaPageData = null;
 	}
 
 	async root(): Promise<MemoryPointer | null> {
-		const pageData = await this.getPage();
+		const pageData = await this.getMetaPage();
 
 		// we seek by 12 bytes since offset is 8 bytes, length is 4 bytes
-		const data = pageData.slice(this.offset, this.offset + 12);
+		const data = pageData.slice(Number(this.offset), Number(this.offset) + 11);
 		const view = new DataView(data);
 
 		const pointerOffset = view.getBigUint64(0);
@@ -32,47 +32,42 @@ export class LinkedMetaPage {
 	}
 
 	async metadata(): Promise<ArrayBuffer> {
-		const pageData = await this.getPage();
+		const pageData = await this.getMetaPage();
 		const lengthData = pageData.slice(
-			this.offset + 24,
-			this.offset + PAGE_SIZE_BYTES
+			Number(this.offset) + 24,
+			Number(this.offset) + PAGE_SIZE_BYTES - 1
 		);
 
 		const lengthView = new DataView(lengthData);
 
 		// read the first four because that represnts length
 		const metadataLength = lengthView.getUint32(0);
 		const metadata = pageData.slice(
-			this.offset + 28,
-			this.offset + metadataLength
+			Number(this.offset) + 28,
+			Number(this.offset) + metadataLength - 1
 		);
 
 		return metadata;
 	}
 
-	private async getPage(): Promise<ArrayBuffer> {
-		if (this.pageData) {
-			return this.pageData;
+	private async getMetaPage(): Promise<ArrayBuffer> {
+		if (this.metaPageData) {
+			return this.metaPageData;
 		}
 
 		const { data } = await this.resolver({
-			start: this.offset,
-			end: this.offset + PAGE_SIZE_BYTES - 1,
+			start: Number(this.offset),
+			end: Number(this.offset) + PAGE_SIZE_BYTES - 1,
 		});
 
-		this.pageData = data;
+		this.metaPageData= data;
 
 		return data;
 	}
 
-	// fillBPTree(data: ArrayBuffer): BPTree {
-	// 	const t = new BPTree(this.resolver, this);
-
-	// }
-
 	async next(): Promise<LinkedMetaPage | null> {
-		const pageData = await this.getPage();
-		const data = pageData.slice(this.offset + 12, this.offset + 12 + 8);
+		const pageData = await this.getMetaPage();
+		const data = pageData.slice(Number(this.offset) + 12, Number(this.offset) + 12 + 7);
 
 		const view = new DataView(data);
 
@@ -83,6 +78,13 @@ export class LinkedMetaPage {
 			return null;
 		}
 
-		return new LinkedMetaPage(this.resolver, Number(nextOffset));
+		return new LinkedMetaPage(this.resolver, nextOffset);
 	}
 }
+
+
+export function ReadMultiBPTree(resolver: RangeResolver, pageFile: PageFile): LinkedMetaPage {
+	const offset = pageFile.page(0);
+
+	return new LinkedMetaPage(resolver, offset);
+}

src/btree/node.ts

@@ -63,7 +63,7 @@ export class BPTreeNode {
 	}
 
 	async unmarshalBinary(buffer: ArrayBuffer): Promise<number> {
-		let dataView = new DataView(buffer.slice(0, 4));
+		let dataView = new DataView(buffer.slice(0, 3));
 		const size = dataView.getUint32(0);
 
 		const leaf = size < 0;
@@ -94,7 +94,7 @@ export class BPTreeNode {
 				const dp = this.keys[idx].dataPointer;
 				const dataSlice = this.data.slice(
 					Number(dp.offset),
-					Number(dp.offset + BigInt(dp.length))
+					Number(dp.offset + BigInt(dp.length)) - 1
 				);
 				this.keys[idx].value = new Uint8Array(dataSlice);
 

src/btree/pagefile.ts

@@ -0,0 +1,34 @@
+import {RangeResolver} from "../resolver";
+
+const pageSizeBytes = 4096;
+
+export class PageFile {
+
+    private resolver: RangeResolver;
+    private pageSize: number = pageSizeBytes;
+
+    constructor(resolver: RangeResolver) {
+        this.resolver = resolver;
+    }
+
+    async readPage(idx: number): Promise<ArrayBuffer> {
+        if(idx < 0) {
+            throw new Error("page cannot be indexed");
+        }
+
+        const start = (idx + 1) * this.pageSize;
+        const end = start + this.pageSize - 1;
+
+        const { data } = await this.resolver({ start, end: end - 1 });
+
+        return data;
+    }
+
+    page(idx: number): bigint {
+        if (idx < 0) {
+            return BigInt(0);
+        }
+
+        return BigInt(idx + 1) * BigInt(this.pageSize);
+    }
+}

src/db/query-validation.ts

@@ -35,7 +35,6 @@ function validateWhere<T extends Schema>(
 		throw new Error("Missing 'where' clause.");
 	}
 
-	console.log("validating where: ", where);
 
 	for (const whereNode of where) {
 		if (!["<", "<=", "==", ">=", ">"].includes(whereNode.operation)) {
@@ -116,7 +115,6 @@ function validateOrderBy<T extends Schema>(
 	whereKey: string
 ): void {
 	if (orderBy) {
-		console.log("validating orderBy: ", orderBy);
 		if (!Array.isArray(orderBy) || orderBy.length === 0) {
 			throw new Error("Invalid 'orderBy' clause.");
 		}
@@ -146,7 +144,6 @@ function validateSelect<T extends Schema>(
 	headers: IndexMeta[]
 ): void {
 	if (select) {
-		console.log("validating select: ", select);
 		if (!Array.isArray(select) || select.length === 0) {
 			throw new Error("Invalid 'select' clause");
 		}

src/index-file/index-file.ts

@@ -1,6 +1,7 @@
-import { LinkedMetaPage } from "../btree/multi";
+import {LinkedMetaPage, ReadMultiBPTree} from "../btree/multi";
 import { LengthIntegrityError, RangeResolver } from "../resolver";
 import { IndexMeta, unmarshalBinaryForIndexMeta } from "./index-meta";
+import {PageFile} from "../btree/pagefile";
 
 
 
@@ -62,9 +63,11 @@ class IndexFileV1<T> implements VersionedIndexFile<T> {
 			return this._tree;
 		}
 
-		this._tree = new LinkedMetaPage(this.resolver, 0);
+		const pageFile = new PageFile(this.resolver);
+		const tree = ReadMultiBPTree(this.resolver, pageFile);
 
-		return this._tree;
+		this._tree = tree;
+		return tree;
 	}
 
 	async metadata(): Promise<FileMeta | null> {

src/tests/multi.test.ts

@@ -0,0 +1,18 @@
+import {RangeResolver} from "../resolver";
+import {PageFile} from "../btree/pagefile";
+import {ReadMultiBPTree} from "../btree/multi";
+
+
+describe("test multi", () => {
+
+    let resolver: RangeResolver;
+
+    it("storing metadata works", async() => {
+        // const pageFile = new PageFile(resolver);
+        //
+        // const tree = ReadMultiBPTree(resolver, pageFile);
+        //
+        // const metadata = await tree.metadata();
+        // console.log(metadata);
+    });
+});

src/tests/node.test.ts

@@ -1,8 +1,11 @@
 import { ReferencedValue } from "../btree/bptree";
 import { BPTreeNode, MemoryPointer } from "../btree/node";
+import fs from 'fs/promises';
+import path from 'path';
+import {RangeResolver} from "../resolver";
 
-const strToUint8Array = (str: string) => {
-	return new Uint8Array(str.split("").map((c) => c.charCodeAt(0)));
+const strToArrayBuffer = (str: string) => {
+	return new Uint8Array(str.split("").map((c) => c.charCodeAt(0))).buffer;
 };
 
 describe("test compare bytes", () => {
@@ -32,15 +35,26 @@ describe("test compare bytes", () => {
 	testCases.forEach(({ a, b, i }, idx) => {
 		it(`test ${idx} compareBytes`, async () => {
 			const result = ReferencedValue.compareBytes(
-				strToUint8Array(a),
-				strToUint8Array(b)
+				strToArrayBuffer(a),
+				strToArrayBuffer(b)
 			);
 			expect(result).toBe(i);
 		});
 	});
 });
 
+
+const readBinaryFile = async (filename: string): Promise<Uint8Array> => {
+	const filePath = path.join(__dirname, filename);
+	const data = await fs.readFile(filePath);
+	return new Uint8Array(data);
+};
+
 describe("node functionality", () => {
+
+	let resolver: RangeResolver;
+
+
 	it("correctly identifies leaf nodes", async () => {
 		const leafKeys = [
 			new ReferencedValue({ offset: BigInt(0), length: 0 }, new Uint8Array()),
@@ -68,4 +82,16 @@ describe("node functionality", () => {
 		);
 		expect(leafNode.pointer(0)).toEqual(leafPointers[0]);
 	});
+
+	it("reads from buffer for leaf node", async() => {
+
+		const buffer = await readBinaryFile('leaf_node_data.bin');
+		console.log(buffer);
+		const node = new BPTreeNode([], [], [], buffer);
+		// const bytesRead = await node.unmarshalBinary();
+		console.log(node)
+
+	});
+
+
 });

src/tests/query-builder.test.ts

@@ -163,7 +163,6 @@ describe("test validate queries", () => {
 
 		// green queries
 		[passQuery, passQuery2].forEach(async (query) => {
-			console.log(query.toQuery());
 			await expect(() => validateQuery(query.toQuery(), headers)).not.toThrow();
 		});
 	});