feat[tidb]:create table if not exist

connector/tidb/datasource_factory.go

@@ -7,6 +7,7 @@ import (
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 	"gorm.io/gorm/logger"
+	"log"
 	"open-indexer/handlers"
 	"open-indexer/model"
 	"os"
@@ -19,6 +20,15 @@ var (
 	once sync.Once
 )
 
+var tblCreateSqlMap = make(map[string]string)
+var initFilePath = "./data/init/"
+
+func init() {
+	tblCreateSqlMap["token_info"] = initFilePath + "create_token_info.sql"
+	tblCreateSqlMap["token_activities"] = initFilePath + "create_token_activities.sql"
+	tblCreateSqlMap["token_balances"] = initFilePath + "create_token_balances.sql"
+}
+
 type Config struct {
 	TiDBUser     string `json:"tidb_user"`
 	TiDBPassword string `json:"tidb_password"`
@@ -56,6 +66,47 @@ func createDB(tidb_user string, tidb_password string, tidb_host string, tidb_por
 	return db, nil
 }
 
+func JudgeTableExistOrNot(db *gorm.DB, tableName string) (bool, error) {
+	var count int
+	db.Raw("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ?", tableName).Scan(&count)
+
+	if count == 0 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func CreateTableIfNotExist[T any](db *gorm.DB, table T, tableName string) error {
+	exist, err := JudgeTableExistOrNot(db, tableName)
+	if !exist {
+		filePath, ok := tblCreateSqlMap[tableName]
+		if !ok {
+			tType := reflect.TypeOf(*new(T))
+			instance := reflect.New(tType).Interface()
+			err := db.AutoMigrate(instance)
+			if err != nil {
+				log.Fatalf("Create table %s failed: %v", tableName, err)
+				return err
+			}
+		} else {
+			content, err := os.ReadFile(filePath)
+			if err != nil {
+				log.Fatalf("Error reading SQL file: %v", err)
+			}
+			sql := string(content)
+			err = db.Exec(sql).Error
+			if err != nil {
+				log.Fatalf("Create table %s failed: %v", tableName, err)
+				return err
+			}
+		}
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func GetDBInstanceByConfigFile(file_name string) (*gorm.DB, error) {
 	var err error
 	once.Do(func() {
@@ -84,11 +135,17 @@ func Upsert[T any](db *gorm.DB, datas []T) error {
 			return err
 		}
 	}
-	logger.Infof("Insert into db successed, items %d %s", len(datas), reflect.TypeOf(*new(T)))
+	logger.Infof("Upsert into db successed, items %d %s", len(datas), reflect.TypeOf(*new(T)))
 	return nil
 }
 
 func ProcessUpsert(db *gorm.DB, inscriptions []*model.Inscription, logEvents []*model.EvmLog, tokens []*model.TokenInfo, tokenActivities []*model.TokenActivity, tokenBalances map[string]map[string]*model.TokenBalance) error {
+	CreateTableIfNotExist(db, model.Inscription{}, model.Inscription{}.TableName())
+	CreateTableIfNotExist(db, model.EvmLog{}, model.EvmLog{}.TableName())
+	CreateTableIfNotExist(db, model.TokenInfo{}, model.TokenInfo{}.TableName())
+	CreateTableIfNotExist(db, model.TokenActivity{}, model.TokenActivity{}.TableName())
+	CreateTableIfNotExist(db, model.TokenBalance{}, model.TokenBalance{}.TableName())
+
 	tx := db.Begin()
 	if tx.Error != nil {
 		return tx.Error

data/init/create_token_activities.sql

@@ -0,0 +1,17 @@
+CREATE TABLE IF NOT EXISTS `token_activities` (
+                                    `block_timestamp` datetime(3) NOT NULL COMMENT 'Timestamp of the block containing the inscription (matches block_timestamp in transactions table)',
+                                    `block_number` bigint(20) NOT NULL COMMENT 'Block number containing the inscription (matches block_number in transactions table)',
+                                    `tx_index` int(11) NOT NULL COMMENT 'Index of the transaction containing the inscription (matches transaction_index in transactions table)',
+                                    `tx_hash` varchar(66) NOT NULL COMMENT 'Unique identifier of the transaction containing the inscription (matches hash in transactions table)',
+                                    `log_index` int(11) NOT NULL COMMENT 'Index of the log within the transaction',
+                                    `type` varchar(255) NOT NULL COMMENT 'mint  transfer  burn',
+                                    `tick` varchar(255) NOT NULL COMMENT 'Token tick',
+                                    `id` varchar(255) NOT NULL COMMENT 'Unique identifier of the inscription',
+                                    `amt` decimal(38, 0) DEFAULT NULL COMMENT 'Mint amount',
+                                    `from_address` varchar(42) DEFAULT NULL COMMENT 'Address sending the inscription (matches from_address in transactions table)',
+                                    `to_address` varchar(42) DEFAULT NULL COMMENT 'Address receiving the inscription (match to_address in transactions table)',
+                                    PRIMARY KEY (
+                                                 `id`, `log_index`, `tx_index`, `tick`,
+                                                 `type`
+                                        )
+);

data/init/create_token_balances.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `token_balances` (
+                                  `block_number` bigint(20) unsigned DEFAULT NULL COMMENT 'Block number containing the transaction',
+                                  `block_timestamp` datetime(3) DEFAULT NULL COMMENT 'Block timestamp containing the transaction',
+                                  `tick` varchar(255) NOT NULL COMMENT 'Token tick',
+                                  `wallet_address` varchar(42) NOT NULL COMMENT 'Address of owner',
+                                  `total_supply` decimal(38, 0) DEFAULT NULL COMMENT 'Max supply',
+                                  `amount` decimal(38, 0) DEFAULT NULL COMMENT 'The balance of wallet balance at the corresponding block height',
+                                  PRIMARY KEY (`tick`, `wallet_address`)
+);

data/init/create_token_info.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS `token_info` (
+    `block_timestamp` datetime(3) NOT NULL COMMENT 'Timestamp of the block containing the inscription (matches block_timestamp in transactions table)',
+    `block_number` bigint(20) NOT NULL COMMENT 'Block number containing the inscription (matches block_number in transactions table)',
+    `tx_index` int(11) NOT NULL COMMENT 'Index of the transaction containing the inscription (matches transaction_index in transactions table)',
+    `tx_hash` varchar(66) NOT NULL COMMENT 'Unique identifier of the transaction containing the inscription (matches hash in transactions table)',
+    `tick` varchar(255) NOT NULL COMMENT 'Token tick',
+    `max_supply` decimal(38, 0) DEFAULT NULL COMMENT 'Max supply',
+    `lim` decimal(38, 0) DEFAULT NULL COMMENT 'Limit of each mint',
+    `wlim` decimal(38, 0) DEFAULT NULL COMMENT 'Limit of each address can maximum mint',
+    `dec` int(11) DEFAULT NULL COMMENT 'Decimal for minimum divie',
+    `creator` varchar(42) DEFAULT NULL COMMENT 'Address originating the inscription (matches from_address in transactions table)',
+    `minted` decimal(38, 0) DEFAULT '0',
+    `holders` decimal(38, 0) DEFAULT '0',
+    `txs` decimal(38, 0) DEFAULT '0',
+    `updated_timestamp` timestamp(3) NULL DEFAULT NULL,
+    `completed_timestamp` timestamp(3) NULL DEFAULT NULL,
+    `id` varchar(255) DEFAULT NULL,
+    PRIMARY KEY (`tick`)
+    );

loader/loader.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"gorm.io/gorm"
 	"log"
+	"open-indexer/connector/tidb"
 	"open-indexer/model"
 	"open-indexer/utils"
 	"os"
@@ -22,7 +23,6 @@ type Holder struct {
 }
 
 func LoadTransactionData(fname string) ([]*model.Transaction, error) {
-
 	file, err := os.Open(fname)
 	if err != nil {
 		return nil, err
@@ -332,8 +332,17 @@ func ConvertAsc20sToTokenActivities(asc20s []*model.Asc20) []*model.TokenActivit
 func LoadTokenInfo(db *gorm.DB) ([]*model.Token, error) {
 	var tokenInfos []*model.TokenInfo
 	var tokens []*model.Token
+	tableName := model.TokenInfo{}.TableName()
+	exist, err := tidb.JudgeTableExistOrNot(db, tableName)
+	if !exist {
+		return tokens, nil
+	}
 
-	err := db.Find(&tokenInfos).Error
+	if err != nil {
+		return tokens, err
+	}
+
+	err = db.Find(&tokenInfos).Error
 	if err != nil {
 		return tokens, err
 	}
@@ -382,7 +391,16 @@ func LoadTokenInfo(db *gorm.DB) ([]*model.Token, error) {
 
 func LoadTokenBalances(db *gorm.DB) ([]*model.TokenBalance, error) {
 	var tokenBalances []*model.TokenBalance
-	err := db.Find(&tokenBalances).Error
+
+	exist, err := tidb.JudgeTableExistOrNot(db, model.TokenBalance{}.TableName())
+	if !exist {
+		return tokenBalances, nil
+	}
+	if err != nil {
+		return tokenBalances, err
+	}
+
+	err = db.Find(&tokenBalances).Error
 	if err != nil {
 		return tokenBalances, err
 	}
@@ -398,7 +416,15 @@ func LoadList(db *gorm.DB) ([]*model.List, error) {
 	var tokenActivities []*model.TokenActivity
 	var lists []*model.List
 
-	err := db.Find(&tokenActivities).Error
+	exist, err := tidb.JudgeTableExistOrNot(db, model.TokenActivity{}.TableName())
+	if !exist {
+		return lists, nil
+	}
+	if err != nil {
+		return lists, err
+	}
+
+	err = db.Find(&tokenActivities).Error
 	if err != nil {
 		return lists, err
 	}

model/evmlog.go

@@ -14,6 +14,10 @@ type EvmLog struct {
 	Timestamp uint64 `gorm:"column:number"`
 }
 
+func (EvmLog) TableName() string {
+	return "evm_logs"
+}
+
 //func NewEvmLogFromMixData(mixData *MixData) *EvmLog {
 //	logEvent := mixData.LogEvent
 //	topic0 := ""

model/inscription.go

@@ -11,3 +11,7 @@ type Inscription struct {
 	ContentType string `gorm:"column:content_type"`
 	Content     string `gorm:"column:content"`
 }
+
+func (Inscription) TableName() string {
+	return "inscriptions"
+}

model/token_activities.go

@@ -15,3 +15,7 @@ type TokenActivity struct {
 	FromAddress    string    `gorm:"column:from_address"`
 	ToAddress      string    `gorm:"column:to_address"`
 }
+
+func (TokenActivity) TableName() string {
+	return "token_activities"
+}

model/token_balances.go

@@ -10,3 +10,7 @@ type TokenBalance struct {
 	TotalSupply    *DDecimal `gorm:"column:total_supply;type:decimal(38,0)"`
 	Amount         *DDecimal `gorm:"column:amount;type:decimal(38,0)"`
 }
+
+func (TokenBalance) TableName() string {
+	return "token_balances"
+}