Skip to content

Commit

Permalink
refactor: factor out AST (#1324)
Browse files Browse the repository at this point in the history
  • Loading branch information
verytactical authored Jan 14, 2025
1 parent e9b6874 commit 8bd13ec
Show file tree
Hide file tree
Showing 59 changed files with 102 additions and 107 deletions.
4 changes: 2 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ Some CLI tests can be found in [.github/workflows/tact.yml](./.github/workflows/

The [src/grammar/grammar.ohm](./src/grammar/grammar.ohm) file contains the Tact grammar expressed in the PEG-like language of the [Ohm.js](https://ohmjs.org) parser generator.

The helper file [src/grammar/grammar.ts](./src/grammar/grammar.ts) contains the logic that transforms concrete syntax trees produced with the help of the Ohm.js-generated parser into abstract syntax trees (ASTs) defined in [src/grammar/ast.ts](./src/grammar/ast.ts). The grammar.ts file also does a bit of grammar validation, like checking that function or constant attributes are not duplicated or that user identifiers do not start with certain reserved prefixes.
The helper file [src/grammar/grammar.ts](./src/grammar/grammar.ts) contains the logic that transforms concrete syntax trees produced with the help of the Ohm.js-generated parser into abstract syntax trees (ASTs) defined in [src/ast/ast.ts](./src/ast/ast.ts). The grammar.ts file also does a bit of grammar validation, like checking that function or constant attributes are not duplicated or that user identifiers do not start with certain reserved prefixes.

The [src/grammar/test](./src/grammar/test) folder contains Tact files that are supposed to be parsed without any issues, and the [src/grammar/test-failed](./src/grammar/test-failed) folder contains grammatically incorrect test files which should result in parser errors. The parser error messages and the locations they point to are fixed in the [src/grammar/**snapshots**/grammar.spec.ts.snap](./src/grammar/__snapshots__/grammar.spec.ts.snap) Jest snapshot file.

Expand Down Expand Up @@ -177,6 +177,6 @@ Some other codegen tests are as follows:

The entry point to the Tact AST pretty-printer is [src/prettyPrinter.ts](./src/prettyPrinter.ts). It is going to be used for the Tact source code formatter once the parser keeps comments and other relevant information.

The AST comparator is defined in [src/grammar/compare.ts](./src/grammar/compare.ts). This is useful, for instance, for static analysis tools which can re-use the Tact TypeScript API.
The AST comparator is defined in [src/ast/compare.ts](./src/ast/compare.ts). This is useful, for instance, for static analysis tools which can re-use the Tact TypeScript API.

The corresponding test spec files can be found in [src/test](./src/test/) folder with the test contracts in [src/test/contracts](./src/test/contracts/) folder.
2 changes: 1 addition & 1 deletion knip.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"entry": ["src/index.ts", "src/node.ts", "bin/tact.js", "bin/unboc.js"],
"project": ["src/**/*.ts", "bin/tact.js", "bin/unboc.js"],
"ignore": [
"src/grammar/ast.ts",
"src/ast/ast.ts",
"src/prettyPrinter.ts",
"src/error/display-to-json.ts",
"src/grammar/src-info.ts",
Expand Down
2 changes: 1 addition & 1 deletion src/abi/AbiFunction.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { AstExpression } from "../grammar/ast";
import { AstExpression } from "../ast/ast";
import { CompilerContext } from "../context/context";
import { WriterContext } from "../generator/Writer";
import { TypeRef } from "../types/types";
Expand Down
2 changes: 1 addition & 1 deletion src/abi/global.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
ensureString,
interpretEscapeSequences,
} from "../optimizer/interpreter";
import { isLiteral } from "../grammar/ast";
import { isLiteral } from "../ast/ast";

export const GlobalFunctions: Map<string, AbiFunction> = new Map([
[
Expand Down
2 changes: 1 addition & 1 deletion src/abi/map.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { writeExpression } from "../generator/writers/writeExpression";
import { throwCompilationError } from "../error/errors";
import { getType } from "../types/resolveDescriptors";
import { AbiFunction } from "./AbiFunction";
import { AstExpression } from "../grammar/ast";
import { AstExpression } from "../ast/ast";

// Helper functions to avoid redundancy
function checkArgumentsLength(
Expand Down
59 changes: 27 additions & 32 deletions src/grammar/ast.ts → src/ast/ast.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Address, Cell, Slice } from "@ton/core";
import { throwInternalCompilerError } from "../error/errors";
import { dummySrcInfo, SrcInfo } from "./src-info";
import { dummySrcInfo, SrcInfo } from "../grammar/src-info";

export type AstModule = {
kind: "module";
Expand Down Expand Up @@ -380,25 +380,28 @@ export type AstBouncedMessageType = {
//

export type AstExpression =
| AstExpressionPrimary
| AstOpBinary
| AstOpUnary
| AstConditional
// AstLiteral could be added in AstExpressionPrimary,
// but AstExpressionPrimary is planned to be removed. See issue #1290 (https://github.com/tact-lang/tact/issues/1290).
| AstLiteral;

export type AstExpressionPrimary =
| AstMethodCall
| AstFieldAccess
| AstStaticCall
| AstStructInstance
| AstId
| AstInitOf
| AstString
| AstLiteral;

export type AstLiteral =
| AstNumber
| AstBoolean
| AstId
| AstNull
| AstInitOf
| AstString;
| AstSimplifiedString
| AstAddress
| AstCell
| AstSlice
| AstCommentValue
| AstStructValue;

export type AstBinaryOperation =
| "+"
Expand Down Expand Up @@ -638,6 +641,20 @@ export type AstBoolean = {
loc: SrcInfo;
};

// An AstSimplifiedString is a string in which escaping characters, like '\\' has been simplified, e.g., '\\' simplified to '\'.
// An AstString is not a literal because it may contain escaping characters that have not been simplified, like '\\'.
// AstSimplifiedString is always produced by the interpreter, never directly by the parser. The parser produces AstStrings, which
// then get transformed into AstSimplifiedString by the interpreter.
export type AstSimplifiedString = {
kind: "simplified_string";
value: string;
id: number;
loc: SrcInfo;
};

/**
* @deprecated AstSimplifiedString
*/
export type AstString = {
kind: "string";
value: string;
Expand All @@ -654,28 +671,6 @@ export type AstNull = {
loc: SrcInfo;
};

export type AstLiteral =
| AstNumber
| AstBoolean
| AstNull
// An AstSimplifiedString is a string in which escaping characters, like '\\' has been simplified, e.g., '\\' simplified to '\'.
// An AstString is not a literal because it may contain escaping characters that have not been simplified, like '\\'.
// AstSimplifiedString is always produced by the interpreter, never directly by the parser. The parser produces AstStrings, which
// then get transformed into AstSimplifiedString by the interpreter.
| AstSimplifiedString
| AstAddress
| AstCell
| AstSlice
| AstCommentValue
| AstStructValue;

export type AstSimplifiedString = {
kind: "simplified_string";
value: string;
id: number;
loc: SrcInfo;
};

export type AstAddress = {
kind: "address";
value: Address;
Expand Down
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion src/grammar/ast-typed.ts → src/ast/getAstSchema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import { Loc } from "@tonstudio/parser-runtime";
import * as A from "./ast";
import { SrcInfo } from "./src-info";
import { SrcInfo } from "../grammar/src-info";

export const getAstSchema = (
factory: A.FactoryAst,
Expand Down
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion src/grammar/rename.ts → src/ast/rename.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
} from "./ast";
import { AstSorter } from "./sort";
import { AstHasher, AstHash } from "./hash";
import { dummySrcInfo } from "./src-info";
import { dummySrcInfo } from "../grammar/src-info";

type GivenName = string;

Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion src/bindings/writeTypescript.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import {
getAllocationOperationFromField,
} from "../storage/allocator";
import { serializers } from "./typescript/serializers";
import { eqNames } from "../grammar/ast";
import { eqNames } from "../ast/ast";

function writeArguments(args: ABIArgument[]) {
const res: string[] = [];
Expand Down
8 changes: 4 additions & 4 deletions src/grammar/store.ts → src/context/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ import {
AstNativeFunctionDecl,
AstTypeDecl,
AstAsmFunctionDef,
} from "./ast";
} from "../ast/ast";
import { throwInternalCompilerError } from "../error/errors";
import { CompilerContext, createContextStore } from "../context/context";
import { ItemOrigin } from "./src-info";
import { Parser } from "./grammar";
import { CompilerContext, createContextStore } from "./context";
import { ItemOrigin } from "../grammar/src-info";
import { Parser } from "../grammar/grammar";

/**
* @public
Expand Down
2 changes: 1 addition & 1 deletion src/error/errors.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import path from "path";
import { cwd } from "process";
import { AstFuncId, AstId, AstTypeId } from "../grammar/ast";
import { AstFuncId, AstId, AstTypeId } from "../ast/ast";
import { SrcInfo } from "../grammar";

export class TactError extends Error {
Expand Down
2 changes: 1 addition & 1 deletion src/generator/createABI.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { ABIGetter, ABIReceiver, ABIType, ContractABI } from "@ton/core";
import { contractErrors } from "../abi/errors";
import { CompilerContext } from "../context/context";
import { idText } from "../grammar/ast";
import { idText } from "../ast/ast";
import { getSupportedInterfaces } from "../types/getSupportedInterfaces";
import { createABITypeRefFromTypeRef } from "../types/resolveABITypeRef";
import { getAllTypes } from "../types/resolveDescriptors";
Expand Down
2 changes: 1 addition & 1 deletion src/generator/writeProgram.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { writeAccessors } from "./writers/writeAccessors";
import { ContractABI } from "@ton/core";
import { writeFunction } from "./writers/writeFunction";
import { calculateIPFSlink } from "../utils/calculateIPFSlink";
import { getRawAST } from "../grammar/store";
import { getRawAST } from "../context/store";
import { emit } from "./emitter/emit";
import {
writeInit,
Expand Down
2 changes: 1 addition & 1 deletion src/generator/writers/id.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { AstId, idText } from "../../grammar/ast";
import { AstId, idText } from "../../ast/ast";

export function funcIdOf(ident: AstId | string): string {
if (typeof ident === "string") {
Expand Down
4 changes: 2 additions & 2 deletions src/generator/writers/resolveFuncType.spec.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { getAstFactory } from "../../grammar/ast";
import { getAstFactory } from "../../ast/ast";
import { resolveDescriptors } from "../../types/resolveDescriptors";
import { WriterContext } from "../Writer";
import { resolveFuncType } from "./resolveFuncType";
import { openContext } from "../../grammar/store";
import { openContext } from "../../context/store";
import { CompilerContext } from "../../context/context";
import { getParser } from "../../grammar";
import { defaultParser } from "../../grammar/grammar";
Expand Down
4 changes: 2 additions & 2 deletions src/generator/writers/writeExpression.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ import {
} from "../../types/resolveDescriptors";
import { WriterContext } from "../Writer";
import { writeExpression } from "./writeExpression";
import { openContext } from "../../grammar/store";
import { openContext } from "../../context/store";
import { resolveStatements } from "../../types/resolveStatements";
import { CompilerContext } from "../../context/context";
import { getParser } from "../../grammar";
import { getAstFactory } from "../../grammar/ast";
import { getAstFactory } from "../../ast/ast";
import { defaultParser } from "../../grammar/grammar";

const code = `
Expand Down
2 changes: 1 addition & 1 deletion src/generator/writers/writeExpression.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
getAstFactory,
idText,
tryExtractPath,
} from "../../grammar/ast";
} from "../../ast/ast";
import {
idTextErr,
TactConstEvalError,
Expand Down
2 changes: 1 addition & 1 deletion src/generator/writers/writeFunction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
idText,
isWildcard,
tryExtractPath,
} from "../../grammar/ast";
} from "../../ast/ast";
import { getType, resolveTypeRef } from "../../types/resolveDescriptors";
import { getExpType } from "../../types/resolveExpression";
import { FunctionDescription, TypeRef } from "../../types/types";
Expand Down
2 changes: 1 addition & 1 deletion src/generator/writers/writeRouter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { ops } from "./ops";
import { resolveFuncType } from "./resolveFuncType";
import { resolveFuncTypeUnpack } from "./resolveFuncTypeUnpack";
import { writeStatement } from "./writeFunction";
import { AstNumber } from "../../grammar/ast";
import { AstNumber } from "../../ast/ast";

export function commentPseudoOpcode(comment: string): string {
return beginCell()
Expand Down
4 changes: 2 additions & 2 deletions src/generator/writers/writeSerialization.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ import {
import { WriterContext } from "../Writer";
import { writeParser, writeSerializer } from "./writeSerialization";
import { writeStdlib } from "./writeStdlib";
import { openContext } from "../../grammar/store";
import { openContext } from "../../context/store";
import { writeAccessors } from "./writeAccessors";
import { getParser } from "../../grammar";
import { getAstFactory } from "../../grammar/ast";
import { getAstFactory } from "../../ast/ast";
import { defaultParser } from "../../grammar/grammar";

const code = `
Expand Down
2 changes: 1 addition & 1 deletion src/grammar/grammar.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { getParser as getParserNext } from "./next";

import { AstExpression, AstImport, AstModule, FactoryAst } from "./ast";
import { AstExpression, AstImport, AstModule, FactoryAst } from "../ast/ast";

import { getParser as getParserPrev } from "./prev/grammar";
import { ItemOrigin } from "./src-info";
Expand Down
2 changes: 1 addition & 1 deletion src/grammar/next/grammar.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { getAstFactory } from "../ast";
import { getAstFactory } from "../../ast/ast";
import { loadCases } from "../../utils/loadCases";
import { getParser } from "../grammar";
import { SrcInfo, isSrcInfo } from "../src-info";
Expand Down
4 changes: 2 additions & 2 deletions src/grammar/next/index.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import * as $ from "@tonstudio/parser-runtime";
import * as A from "../ast";
import * as A from "../../ast/ast";
import * as G from "./grammar";
import type { $ast } from "./grammar";
import {
TactCompilationError,
throwInternalCompilerError,
} from "../../error/errors";
import { SyntaxErrors, syntaxErrorSchema } from "../parser-error";
import { AstSchema, getAstSchema } from "../ast-typed";
import { AstSchema, getAstSchema } from "../../ast/getAstSchema";
import { getSrcInfo, ItemOrigin } from "../src-info";
import { displayToString } from "../../error/display-to-string";
import { makeMakeVisitor } from "../../utils/tricks";
Expand Down
2 changes: 1 addition & 1 deletion src/grammar/prev/grammar.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { getAstFactory } from "../ast";
import { getAstFactory } from "../../ast/ast";
import { loadCases } from "../../utils/loadCases";
import { getParser } from "../grammar";
import { SrcInfo, isSrcInfo } from "../src-info";
Expand Down
2 changes: 1 addition & 1 deletion src/grammar/prev/grammar.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import {
AstNumberBase,
AstId,
FactoryAst,
} from "../ast";
} from "../../ast/ast";
import { ItemOrigin, SrcInfo } from "../src-info";
import { displayToString } from "../../error/display-to-string";
import { ParserErrors, parserErrorSchema } from "./parser-error";
Expand Down
2 changes: 1 addition & 1 deletion src/grammar/test/expr-equality.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { eqExpressions, getAstFactory } from "../ast";
import { eqExpressions, getAstFactory } from "../../ast/ast";
import { getParser } from "../";
import { defaultParser } from "../grammar";

Expand Down
2 changes: 1 addition & 1 deletion src/grammar/test/expr-is-value.spec.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//type Test = { expr: string; isValue: boolean };

import { getAstFactory, isLiteral } from "../ast";
import { getAstFactory, isLiteral } from "../../ast/ast";
import { getParser } from "../";
import { defaultParser } from "../grammar";

Expand Down
2 changes: 1 addition & 1 deletion src/imports/resolveImports.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { resolveImports } from "./resolveImports";
import { createNodeFileSystem } from "../vfs/createNodeFileSystem";
import path from "path";
import { getParser } from "../grammar";
import { getAstFactory } from "../grammar/ast";
import { getAstFactory } from "../ast/ast";
import { defaultParser } from "../grammar/grammar";

describe("resolveImports", () => {
Expand Down
8 changes: 4 additions & 4 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ export {
projectSchema,
configSchema,
} from "./config/parseConfig";
export { AstSorter } from "./grammar/sort";
export { AstRenamer } from "./grammar/rename";
export { AstHasher } from "./grammar/hash";
export { AstComparator } from "./grammar/compare";
export { AstSorter } from "./ast/sort";
export { AstRenamer } from "./ast/rename";
export { AstHasher } from "./ast/hash";
export { AstComparator } from "./ast/compare";

export {
Config,
Expand Down
2 changes: 1 addition & 1 deletion src/optimizer/algebraic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
AstOpUnary,
eqExpressions,
isLiteral,
} from "../grammar/ast";
} from "../ast/ast";
import { ExpressionTransformer, Rule } from "./types";
import {
checkIsBinaryOpNode,
Expand Down
2 changes: 1 addition & 1 deletion src/optimizer/associative.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {
AstLiteral,
AstOpBinary,
isLiteral,
} from "../grammar/ast";
} from "../ast/ast";
import * as iM from "./interpreter";
import { ExpressionTransformer, Rule } from "./types";
import {
Expand Down
2 changes: 1 addition & 1 deletion src/optimizer/constEval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
AstUnaryOperation,
isLiteral,
AstLiteral,
} from "../grammar/ast";
} from "../ast/ast";
import {
TactConstEvalError,
throwInternalCompilerError,
Expand Down
Loading

0 comments on commit 8bd13ec

Please sign in to comment.