record token position to ast node, optimize error message #13 (#14)

.gitignore

@@ -1,6 +1,5 @@
 pkg/parser/y.output
 *.pdf
-ppl
-ppl.exe
 rl.sh
-dist/
+dist/
+test.output

Makefile

@@ -1,7 +1,40 @@
 
-.PHONY: lint
+.PHONY: lint test tools
 
-GOLINT_BINARY = golangci-lint
+GOLINT_BINARY := golangci-lint
+
+DATE := $(shell date -u +'%Y-%m-%d %H:%M:%S')
+
+VERSION := $(shell git describe --always --tags)
 
 lint:
 	$(GOLINT_BINARY) run --fix --allow-parallel-runners
+
+test:
+	@truncate -s 0 test.output
+	@echo "#####################" | tee -a test.output
+	@echo "#" $(DATE) | tee -a test.output
+	@echo "#" $(VERSION) | tee -a test.output
+	@echo "#####################" | tee -a test.output
+	i=0; \
+	for pkg in `go list ./...`; do \
+		echo "# testing $$pkg..." | tee -a test.output; \
+		CGO_ENABLED=1 LOGGER_PATH=nul go test -timeout 1m -cover $$pkg; \
+		if [ $$? != 0 ]; then \
+			printf "\033[31m [FAIL] %s\n\033[0m" $$pkg; \
+			i=`expr $$i + 1`; \
+		else \
+			echo "######################"; \
+			fi \
+	done; \
+	if [ $$i -gt 0 ]; then \
+		printf "\033[31m %d case failed.\n\033[0m" $$i; \
+		exit 1; \
+	else \
+		printf "\033[32m all testinig passed.\n\033[0m"; \
+	fi
+
+tools:
+	mkdir -p ./dist
+	rm -r ./dist/
+	go build -o ./dist/ cmd/ppl/ppl.go 

cmd/ppl/ppl.go

@@ -11,6 +11,7 @@ import (
 	plruntime "github.com/GuanceCloud/ppl/pkg/engine/runtime"
 	"github.com/GuanceCloud/ppl/pkg/inimpl/guancecloud/funcs"
 	"github.com/GuanceCloud/ppl/pkg/inimpl/guancecloud/input"
+
 	"github.com/influxdata/influxdb1-client/models"
 	influxdb "github.com/influxdata/influxdb1-client/v2"
 	"github.com/spf13/cobra"
@@ -176,9 +177,9 @@ func runcScript(options *Option) {
 	tn = pt.Time
 	measurement = pt.Measurement
 
-	err = engine.RunScriptWithRMapIn(script, pt, nil)
-	if err != nil {
-		fmt.Println(err)
+	errR := engine.RunScriptWithRMapIn(script, pt, nil)
+	if errR != nil {
+		fmt.Println(errR)
 		os.Exit(1)
 	}
 

pkg/ast/ast.go

@@ -10,21 +10,20 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+
+	"github.com/GuanceCloud/ppl/pkg/token"
 )
 
 type NodeType uint
 
-func (NodeType) String() string {
-	return ""
-}
-
 const (
 	// expr.
 	TypeInvaild NodeType = iota
 
 	TypeIdentifier
 	TypeStringLiteral
-	TypeNumberLiteral
+	TypeIntegerLiteral
+	TypeFloatLiteral
 	TypeBoolLiteral
 	TypeNilLiteral
 
@@ -43,13 +42,64 @@ const (
 	TypeCallExpr
 
 	// stmt.
+	TypeBlockStmt
 	TypeIfelseStmt
 	TypeForStmt
 	TypeForInStmt
 	TypeContinueStmt
 	TypeBreakStmt
 )
 
+func (t NodeType) String() string {
+	switch t {
+	case TypeInvaild:
+		return "Invaild"
+	case TypeIdentifier:
+		return "Identifier"
+	case TypeStringLiteral:
+		return "StringLiteral"
+	case TypeIntegerLiteral:
+		return "IntLiteral"
+	case TypeFloatLiteral:
+		return "FloatLiteral"
+	case TypeBoolLiteral:
+		return "BoolLiteral"
+	case TypeNilLiteral:
+		return "NilLiteral"
+	case TypeListInitExpr:
+		return "ListInitExpr"
+	case TypeMapInitExpr:
+		return "MapInitExpr"
+	case TypeParenExpr:
+		return "ParenExpr"
+	case TypeAttrExpr:
+		return "AttrExpr"
+	case TypeIndexExpr:
+		return "IndexExpr"
+	case TypeArithmeticExpr:
+		return "ArithmeticExpr"
+	case TypeConditionalExpr:
+		return "ConditionalExpr"
+	case TypeAssignmentExpr:
+		return "AssignmentExpr"
+	case TypeCallExpr:
+		return "CallExpr"
+	case TypeBlockStmt:
+		return "BlockStmt"
+	case TypeIfelseStmt:
+		return "IfelseStmt"
+	case TypeForStmt:
+		return "ForStmt"
+	case TypeForInStmt:
+		return "ForInStmt"
+	case TypeContinueStmt:
+		return "ContinueStmt"
+	case TypeBreakStmt:
+		return "BreakStmt"
+	}
+	return "Undefined"
+}
+
 type Stmts []*Node
 
 type KwArgs map[string]*Node
@@ -85,10 +135,11 @@ type Node struct {
 	// expr
 	Identifier *Identifier
 
-	StringLiteral *StringLiteral
-	NumberLiteral *NumberLiteral
-	BoolLiteral   *BoolLiteral
-	NilLiteral    *NilLiteral
+	StringLiteral  *StringLiteral
+	IntegerLiteral *IntegerLiteral
+	FloatLiteral   *FloatLiteral
+	BoolLiteral    *BoolLiteral
+	NilLiteral     *NilLiteral
 
 	ListInitExpr *ListInitExpr
 	MapInitExpr  *MapInitExpr
@@ -105,8 +156,9 @@ type Node struct {
 	CallExpr *CallExpr
 
 	// stmt
-	IfelseStmt *IfelseStmt
+	BlockStmt *BlockStmt
 
+	IfelseStmt   *IfelseStmt
 	ForStmt      *ForStmt
 	ForInStmt    *ForInStmt
 	ContinueStmt *ContinueStmt
@@ -119,8 +171,10 @@ func (node *Node) String() string {
 		return node.Identifier.String()
 	case TypeStringLiteral:
 		return node.StringLiteral.String()
-	case TypeNumberLiteral:
-		return node.NumberLiteral.String()
+	case TypeIntegerLiteral:
+		return node.IntegerLiteral.String()
+	case TypeFloatLiteral:
+		return node.FloatLiteral.String()
 	case TypeBoolLiteral:
 		return node.BoolLiteral.String()
 	case TypeNilLiteral:
@@ -171,10 +225,17 @@ func WrapStringLiteral(node *StringLiteral) *Node {
 	}
 }
 
-func WrapNumberLiteral(node *NumberLiteral) *Node {
+func WrapIntegerLiteral(node *IntegerLiteral) *Node {
+	return &Node{
+		NodeType:       TypeIntegerLiteral,
+		IntegerLiteral: node,
+	}
+}
+
+func WrapFloatLiteral(node *FloatLiteral) *Node {
 	return &Node{
-		NodeType:      TypeNumberLiteral,
-		NumberLiteral: node,
+		NodeType:     TypeFloatLiteral,
+		FloatLiteral: node,
 	}
 }
 
@@ -289,3 +350,80 @@ func WrapBreakStmt(node *BreakStmt) *Node {
 		BreakStmt: node,
 	}
 }
+
+func WrapeBlockStmt(node *BlockStmt) *Node {
+	return &Node{
+		NodeType:  TypeBlockStmt,
+		BlockStmt: node,
+	}
+}
+
+func (node *Node) StartPos() token.Pos {
+	return NodeStartPos(node)
+}
+
+func NodeStartPos(node *Node) token.Pos {
+	if node == nil {
+		return -1
+	}
+	switch node.NodeType {
+	case TypeInvaild:
+		return -1
+	case TypeIdentifier:
+		return node.Identifier.Start
+	case TypeStringLiteral:
+		return node.StringLiteral.Start
+	case TypeIntegerLiteral:
+		return node.IntegerLiteral.Start
+	case TypeFloatLiteral:
+		return node.FloatLiteral.Start
+	case TypeBoolLiteral:
+		return node.BoolLiteral.Start
+	case TypeNilLiteral:
+		return node.NilLiteral.Start
+
+	case TypeListInitExpr:
+		return node.ListInitExpr.LBracket
+	case TypeMapInitExpr:
+		return node.MapInitExpr.LBrace
+
+	case TypeParenExpr:
+		return node.ParenExpr.LParen
+
+	case TypeAttrExpr:
+		return node.AttrExpr.Start
+
+	case TypeIndexExpr:
+		return node.IndexExpr.Obj.Start
+
+	case TypeArithmeticExpr:
+		return node.ArithmeticExpr.LHS.StartPos()
+	case TypeConditionalExpr:
+		return node.ConditionalExpr.LHS.StartPos()
+	case TypeAssignmentExpr:
+		return node.AssignmentExpr.LHS.StartPos()
+
+	case TypeCallExpr:
+		return node.CallExpr.NamePos
+
+	case TypeBlockStmt:
+		return node.BlockStmt.LBracePos
+
+	case TypeIfelseStmt:
+		if len(node.IfelseStmt.IfList) > 0 {
+			return node.IfelseStmt.IfList[0].Start
+		} else {
+			return -1
+		}
+
+	case TypeForStmt:
+		return node.ForStmt.ForPos
+	case TypeForInStmt:
+		return node.ForInStmt.ForPos
+	case TypeContinueStmt:
+		return node.ContinueStmt.Start
+	case TypeBreakStmt:
+		return node.BreakStmt.Start
+	}
+	return -1
+}

pkg/ast/dtype.go

@@ -5,6 +5,8 @@
 
 package ast
 
+import "github.com/spf13/cast"
+
 type DType uint
 
 func (t DType) String() string {
@@ -56,31 +58,16 @@ func DectDataType(val any) (any, DType) {
 		return nil, Nil
 	}
 
-	switch val := val.(type) {
+	switch val.(type) {
 	case string:
 		return val, String
-	case int:
-		return int64(val), Int
-	case int16:
-		return int64(val), Int
-	case int32:
-		return int64(val), Int
-	case int8:
-		return int64(val), Int
-	case uint:
-		return int64(val), Int
-	case uint16:
-		return int64(val), Int
-	case uint32:
-		return int64(val), Int
-	case uint64:
-		return int64(val), Int
-	case uint8:
-		return int64(val), Int
+	case int, int16, int32, int8,
+		uint, uint16, uint32, uint64, uint8:
+		return cast.ToInt64(val), Int
 	case int64:
 		return val, Int
 	case float32:
-		return float64(val), Float
+		return cast.ToFloat64(val), Float
 	case float64:
 		return val, Float
 	case bool: