Add parsing of numbers to tokeniser in VxBuild reimpl

Voxellius · Aug 13, 2024 · 46a506b · 46a506b
1 parent 5250d58
commit 46a506b
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 8 deletions.
diff --git a/examples/test.vxl b/examples/test.vxl
@@ -1 +1 @@
-'test2'
+"test2" 123.56e+7
diff --git a/stdlib/io/io.vxl b/stdlib/io/io.vxl
@@ -73,7 +73,7 @@ function readFile(path, start, end) {
     var fileSize = syscall io_seek(file, -1, false);
 
     if (end >= fileSize) {
-        end = fileSize - 1;
+        end = fileSize;
     }
 
     var bytesToRead = end - start;

diff --git a/test/stdlib-io/expected.log b/test/stdlib-io/expected.log
@@ -1,4 +1,4 @@
 Hello, world!
 File size: 20
-File contents: This is a test file
-Written file contents: This is a test
+File contents: This is a test file.
+Written file contents: This is a test!
diff --git a/tools/vxbuild/tokeniser.vxl b/tools/vxbuild/tokeniser.vxl
@@ -67,6 +67,7 @@ export function tokenise(sourceContainer) {
 
     var mutables = {
         match: null,
+        matchedString: null,
         source: sourceContainer.source,
         previousPosition: 0,
         currentPosition: 0
@@ -80,6 +81,7 @@ export function tokenise(sourceContainer) {
         var match = mutables.match = patterns.match(mutables.source, pattern, {matchRest: true});
 
         if (match.matches) {
+            mutables.matchedString = mutables.source.substring(0, match.index);
             mutables.source = mutables.source.substring(match.index);
             mutables.previousPosition = mutables.currentPosition;
             mutables.currentPosition += match.index;
@@ -91,7 +93,7 @@ export function tokenise(sourceContainer) {
     }
 
     function addToken(tokenClass, value) {
-        value ??= mutables.source.substring(0, mutables.match.index);
+        value ??= mutables.matchedString;
 
         var token = new tokenClass(value);
 
@@ -149,7 +151,7 @@ export function tokenise(sourceContainer) {
             continue;
         }
 
-        if (matchToken("\*")) {
+        if (matchToken("/*")) {
             blockCommentDepth++;
             continue;
         }
@@ -161,7 +163,7 @@ export function tokenise(sourceContainer) {
             }
 
             matchToken([
-                any([
+                patterns.any([
                     patterns.repeat(1, infinity, [
                         patterns.anyCharExcept("*/")
                     ]),
@@ -336,7 +338,44 @@ export function tokenise(sourceContainer) {
             continue;
         }
 
-        // TODO: Add all other syntax features
+        if (matchToken([
+            patterns.any([patterns.ALPHA, patterns.anyChar("$_")]),
+            patterns.repeat(0, infinity, [
+                patterns.any([patterns.ALPHANUMERIC, patterns.anyChar("$_")])
+            ])
+        ])) {
+            addToken(IdentifierToken);
+            continue;
+        }
+
+        if (matchToken([
+            patterns.any([
+                patterns.pattern([
+                    patterns.repeat(1, infinity, [patterns.DIGIT]),
+                    patterns.maybe([
+                        patterns.string("."),
+                        patterns.repeat(0, infinity, [patterns.DIGIT])
+                    ])
+                ]),
+                patterns.pattern([
+                    patterns.repeat(0, infinity, [patterns.DIGIT]),
+                    patterns.string("."),
+                    patterns.maybe([
+                        patterns.repeat(1, infinity, [patterns.DIGIT])
+                    ])
+                ])
+            ]),
+            patterns.maybe([
+                patterns.anyChar("eE"),
+                patterns.maybe([patterns.anyChar("+-")]),
+                patterns.repeat(1, infinity, [patterns.DIGIT])
+            ])
+        ])) {
+            addToken(NumberToken, mutables.matchedString.toNumber());
+            continue;
+        }
+
+        // TODO: Allow base conversions in Voxel to parse binary, octal and hexadecimal numbers
 
         if (matchToken([patterns.WHITESPACE_STRING])) {
             continue;