Add basic expression tokeniser for VxBuild reimpl

examples/test.vxl

@@ -1 +1 @@
-"test2" 123.56e+7
+true

tools/vxbuild-js/expressions.js

@@ -109,7 +109,7 @@ export class ThisNode extends ast.AstNode {
     generateCode(options) {
         return codeGen.bytes(codeGen.vxcTokens.THIS);
     }
-};
+}
 
 export class SuperNode extends ast.AstNode {
     static HUMAN_READABLE_NAME = "`super`";

tools/vxbuild-js/statements.js

@@ -76,7 +76,7 @@ export class StatementBlockNode extends ast.AstNode {
                     continue;
                 }
 
-                throw new sources.SourceError("Expected statement or \`}\`", tokens[0]?.sourceContainer, tokens[0]?.location);
+                throw new sources.SourceError("Expected statement or `}`", tokens[0]?.sourceContainer, tokens[0]?.location);
             }
         } else {
             instance.expectChildByMatching(tokens, [StatementNode], namespace);
@@ -261,8 +261,6 @@ export class IfStatementNode extends ast.AstNode {
         );
 
         return codeGen.join(
-            // skipTrueDefinitionCode,
-            // skipFalseDefinitionCode,
             notConditionCode,
             skipTrueCode,
             isTrueCode,

tools/vxbuild/expressions.vxl

@@ -0,0 +1,163 @@
+import "sources.vxl" as sources;
+import "namespaces.vxl" as namespaces;
+import "tokeniser.vxl" as tokeniser;
+import "ast.vxl" as ast;
+import "parser.vxl" as parser;
+import "statements.vxl" as statements;
+
+enum ExpressionError {
+    NOT_IMPLEMENTED
+}
+
+class ThisNode extends ast.AstNode {
+    HUMAN_READABLE_NAME = "`this`";
+
+    MATCH_QUERIES = [
+        new ast.TokenQuery(tokeniser.KeywordToken, "this")
+    ];
+
+    create(tokens, namespace) {
+        var instance = new ThisNode();
+
+        this.eat(tokens);
+
+        return instance;
+    }
+}
+
+class ThingNode extends ast.AstNode {
+    HUMAN_READABLE_NAME = "thing expression";
+
+    MATCH_QUERIES = [
+        new ast.TokenQuery(tokeniser.AtomToken),
+        new ast.TokenQuery(tokeniser.TypeNameToken),
+        new ast.TokenQuery(tokeniser.IdentifierToken),
+        new ast.TokenQuery(tokeniser.StringToken),
+        new ast.TokenQuery(tokeniser.NumberToken)
+    ];
+
+    value = null;
+    containedValueTokens = [];
+    containerType = null;
+    memoisedContainedTypeValue = null;
+    bufferLength = null;
+    enumIdentifier = null;
+    enumEntryIdentifier = null;
+    enumNamespace = null;
+
+    create(tokens, namespace) {
+        var instance = new ThingNode();
+
+        var token = this.eat(tokens);
+
+        if (token is tokeniser.AtomToken) {
+            instance.value = {
+                "true": true,
+                "false": false,
+                "null": null,
+                "infinity": infinity,
+                "nan": nan
+            }[token.value];
+        } else if (token is StringToken || token is NumberToken) {
+            instance.value = token.value;
+        } else {
+            // TODO: Implement all others
+            throw ExpressionError.NOT_IMPLEMENTED;
+        }
+
+        return instance;
+    }
+
+    createByValue(value) {
+        var instance = new ThingNode();
+
+        instance.value = value;
+
+        return instance;
+    }
+}
+
+class ExpressionNode extends ast.AstNode {
+    HUMAN_READABLE_NAME = "expression";
+
+    MATCH_QUERIES = [
+        new ast.TokenQuery(tokeniser.KeywordToken, "var")
+    ]
+        .concat(ThisNode.MATCH_QUERIES)
+        .concat(ThingNode.MATCH_QUERIES)
+    ;
+
+    create(tokens, namespace) {
+        var instance = new ExpressionNode();
+
+        instance.expectChildByMatching(tokens, [ExpressionLeafNode], namespace); // TODO: Expect ternary operator instead
+
+        return instance;
+    }
+
+    estimateTruthiness() {
+        if (this.children == []) {
+            return null;
+        }
+
+        return this.children[0].estimateTruthiness();
+    }
+}
+
+class ExpressionLeafNode extends ExpressionNode {
+    matches(options) {
+        return true; // TODO: Remove (temporary for testing)
+    }
+
+    create(tokens, namespace) {
+        var instance = new ExpressionLeafNode();
+        var assigningToLocalVariable = false;
+
+        // TODO: Check unary operators here
+        // TODO: Check system call node here
+
+        if (this.maybeEat(tokens, [new ast.TokenQuery(tokeniser.KeywordToken, "var")])) {
+            assigningToLocalVariable = true;
+        }
+
+        if (false) { // FIXME: Fix issue with `else` and match opening bracket
+            instance.expectChildByMatching(tokens, [ExpressionNode], namespace);
+
+            this.eat(tokens, [new ast.TokenQuery(tokeniser.BracketToken, ")")]);
+
+            // TODO: Add accessors here
+        } else {
+            instance.expectChildByMatching(tokens, [ExpressionThingNode], namespace);
+
+            instance.children = instance.children[0].children;
+        }
+
+        // TODO: Check for assignment here
+
+        return instance;
+    }
+
+    estimateTruthiness() {
+        return this.children[-1].estimateTruthiness();
+    }
+}
+
+class ExpressionThingNode extends ExpressionLeafNode {
+    MATCH_QUERIES = []
+        .concat(ThisNode.MATCH_QUERIES)
+        .concat(ThingNode.MATCH_QUERIES)
+    ;
+
+    create(tokens, namespace) {
+        var instance = new ExpressionThingNode();
+
+        instance.expectChildByMatching(tokens, [
+            ThisNode,
+            ThingNode
+        ], namespace);
+
+        // TODO: Add accessors here
+
+        return instance;
+    }
+}

tools/vxbuild/namespaces.vxl

@@ -488,7 +488,7 @@ function propertyIsUsed(propertySymbol) {
 
 function init() {
     var location = paths.resolve(common.STDLIB_DIR, "core.vxl");
-    var source = "'test'"; // io.readFile(location).toString();
+    var source = "true"; // io.readFile(location).toString();
     var sourceContainer = new sources.SourceContainer(source, location);
 
     coreNamespace = new Namespace(sourceContainer, "#core");

tools/vxbuild/parser.vxl

@@ -1,6 +1,6 @@
 import "tokeniser.vxl" as tokeniser;
 import "namespaces.vxl" as namespaces;
-// import "statements.vxl" as statements; // TODO: Implement this
+import "statements.vxl" as statements;
 import "ast.vxl" as ast;
 
 class ModuleNode extends ast.AstNode {
@@ -14,17 +14,15 @@ class ModuleNode extends ast.AstNode {
         var instance = new ModuleNode();
 
         while (true) {
-            if (tokens == []) {
+            if (tokens.length == 0) {
                 break;
             }
 
             if (this.maybeEat(tokens, [new ast.TokenQuery(tokeniser.StatementDelimeterToken)])) {
                 continue;
             }
 
-            tokens.shift();
-
-            // instance.expectChildByMatching(tokens, [statements.StatementNode], namespace); // TODO: Implement this
+            instance.expectChildByMatching(tokens, [statements.StatementNode], namespace);
         }
 
         return instance;

tools/vxbuild/statements.vxl

@@ -0,0 +1,60 @@
+import paths;
+
+import "common.vxl" as common;
+import "sources.vxl" as sources;
+import "tokeniser.vxl" as tokeniser;
+import "namespaces.vxl" as namespaces;
+import "ast.vxl" as ast;
+import "expressions.vxl" as expressions;
+
+class StatementNode extends ast.AstNode {
+    HUMAN_READABLE_NAME = "statement";
+
+    matches(options) {
+        return true;
+    }
+
+    create(tokens, namespace) {
+        var instance = new StatementNode();
+
+        instance.expectChildByMatching(tokens, [
+            expressions.ExpressionNode
+        ], namespace);
+
+        return instance;
+    }
+}
+
+class StatementBlockNode extends ast.AstNode {
+    HUMAN_READABLE_NAME = "statement block";
+
+    matches(options) {
+        return true;
+    }
+
+    create(tokens, namespace) {
+        var instance = new StatementBlockNode();
+
+        if (this.maybeEat(tokens, [new ast.TokenQuery(tokeniser.BracketToken, "{")])) {
+            while (true) {
+                if (this.maybeEat(tokens, [new ast.TokenQuery(tokeniser.BracketToken, "}")])) {
+                    break;
+                }
+
+                if (this.maybeEat(tokens, [new ast.TokenQuery(tokeniser.StatementDelimeterToken)])) {
+                    continue;
+                }
+
+                if (instance.addChildByMatching(tokens, [StatementNode], namespace)) {
+                    continue;
+                }
+
+                throw new sources.SourceError("Exepcted statement or `}`", tokens[0].sourceContainer, tokens[0].location);
+            }
+        } else {
+            instance.expectChildByMatching(tokens, [StatementNode], namespace);
+        }
+
+        return instance;
+    }
+}