-
Notifications
You must be signed in to change notification settings - Fork 280
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #63 from bufanyun/v2.0
up
- Loading branch information
Showing
13 changed files
with
6,112 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
package storager | ||
|
||
import ( | ||
"crypto/md5" | ||
"errors" | ||
"fmt" | ||
"io" | ||
"os" | ||
"testing" | ||
) | ||
|
||
func TestMergePartFile(t *testing.T) { | ||
srcFile, _ := os.Open("1.zip") | ||
defer srcFile.Close() | ||
fileInfo, err := srcFile.Stat() | ||
if err != nil { | ||
t.Error(err) | ||
} | ||
hash := md5.New() | ||
_, err = io.Copy(hash, srcFile) | ||
sum := hash.Sum(nil) | ||
fileHash := fmt.Sprintf("%x", sum) | ||
|
||
var singleSize = int64(1024 * 1024 * 100) | ||
chunks := int(fileInfo.Size() / singleSize) | ||
if fileInfo.Size()%singleSize != 0 { | ||
chunks += 1 | ||
} | ||
srcFile.Seek(0, io.SeekStart) | ||
for j := 0; j < chunks; j++ { | ||
partSize := singleSize | ||
if j == chunks-1 { | ||
partSize = fileInfo.Size() - int64(j)*singleSize | ||
} | ||
partData := make([]byte, partSize) | ||
_, err = io.ReadFull(srcFile, partData) | ||
pf, _ := os.Create(fmt.Sprintf("tmp/%d", j+1)) | ||
_, err = pf.Write(partData) | ||
if err != nil { | ||
t.Error(err) | ||
return | ||
} | ||
pf.Close() | ||
} | ||
|
||
err = MergePartFile("tmp/", "2.zip") | ||
if err != nil { | ||
t.Error(err) | ||
return | ||
} | ||
|
||
f2, _ := os.Open("2.zip") | ||
hash2 := md5.New() | ||
_, err = io.Copy(hash2, f2) | ||
sum2 := hash.Sum(nil) | ||
fileHash2 := fmt.Sprintf("%x", sum2) | ||
if fileHash != fileHash2 { | ||
t.Error(errors.New("hash mismatch")) | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
# 数据库补丁支持 | ||
|
||
> 将用于抹平不同数据库的差异性,以支撑基于数据库的部分特性。 | ||
|
||
## 特性 | ||
|
||
- 增加对数据库表字段注释的支持,特别是 sqlite 的注释支持。 | ||
- 增加对数据库表名注释的支持,特别是 sqlite 的注释支持。 | ||
|
||
> 注意以上支持中,由于 sqlite 的特殊性,对 sqlite 表创建的 sql 语句有所要求,具体示例请参考 [sqlite_example.sql](./sqlite_example.sql) | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
package db | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
|
||
"github.com/gogf/gf/v2/database/gdb" | ||
"github.com/gogf/gf/v2/frame/g" | ||
) | ||
|
||
// 获取数据库表字段及注释 | ||
// usage: | ||
// | ||
// fields, err := db.GetFieldsWithComment(ctx, in.Table, in.Name) | ||
// if err != nil { | ||
// return | ||
// } | ||
// for _, v := range fields {} | ||
func GetFieldsWithComment(ctx context.Context, tableName, dbTag string) (fields map[string]*gdb.TableField, err error) { | ||
db := g.DB(dbTag) | ||
fields, err = db.TableFields(ctx, tableName) // 使用 goframe 框架本身已完美支持 mysql 获取表字段及注释 | ||
dbConf := db.GetConfig() | ||
switch dbConf.Type { | ||
case "sqlite": | ||
fields, err = fixSqliteFieldsComment(ctx, tableName, db, fields) | ||
} | ||
return | ||
} | ||
|
||
type TableComment struct { | ||
Name string `json:"name"` | ||
Comment string `json:"comment"` | ||
} | ||
|
||
// 获取数据库表字段及注释 | ||
func GetTablesWithComment(ctx context.Context, dbTag string) (tables []*TableComment, err error) { | ||
db := g.DB(dbTag) | ||
dbConf := db.GetConfig() | ||
switch dbConf.Type { | ||
case "mysql": | ||
sql := "SELECT TABLE_NAME as name, TABLE_COMMENT as comment FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '%s'" | ||
if err = db.Ctx(ctx).Raw(fmt.Sprintf(sql, dbConf.Name)).Scan(&tables); err != nil { | ||
return | ||
} | ||
case "sqlite": | ||
var tableNames []string | ||
tableNames, err = db.Tables(ctx) | ||
if err != nil { | ||
return | ||
} | ||
tables, err = transSqliteTablesComment(ctx, tableNames, db) | ||
} | ||
return | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
CREATE TABLE IF NOT EXISTS `user1` ( -- 用户管理 | ||
`id` INTEGER, -- 编号 | ||
`name` VARCHAR(50), -- 名称 | ||
`email` VARCHAR(255), -- 邮箱 | ||
`address` TEXT, -- 地址 | ||
`salt` VARCHAR(50), -- 盐 | ||
`password` VARCHAR(50), -- 密码 | ||
`mark` VARCHAR(255), -- 备注 | ||
`permission` TEXT, -- 权限 | ||
`created_user_id` INTEGER, -- 创建者编号 | ||
`created_at` DATETIME, -- 创建时间 | ||
`updated_at` DATETIME, -- 更新时间 | ||
`deleted_at` DATETIME, -- 删除时间 | ||
PRIMARY KEY(`id`) | ||
); | ||
CREATE TABLE "user2" ( -- 用户管理 | ||
"id" INTEGER, -- 编号 | ||
"name" VARCHAR(50), -- 名称 | ||
"email" VARCHAR(255), -- 邮箱 | ||
"address" TEXT, -- 地址 | ||
"salt" VARCHAR(50), -- 盐 | ||
"password" VARCHAR(50), -- 密码 | ||
"mark" VARCHAR(255), -- 备注 | ||
"permission" TEXT, -- 权限 | ||
"created_user_id" INTEGER, -- 创建者编号 | ||
"created_at" DATETIME, -- 创建时间 | ||
"updated_at" DATETIME, -- 更新时间 | ||
"deleted_at" DATETIME, -- 删除时间 | ||
PRIMARY KEY("id") | ||
); | ||
CREATE TABLE IF NOT EXISTS user3 ( -- 用户管理 | ||
id INTEGER, -- 编号 | ||
name VARCHAR(50), -- 名称 | ||
email VARCHAR(255), -- 邮箱 | ||
address TEXT, -- 地址 | ||
salt VARCHAR(50), -- 盐 | ||
password VARCHAR(50), -- 密码 | ||
mark VARCHAR(255), -- 备注 | ||
permission TEXT, -- 权限 | ||
created_user_id INTEGER, -- 创建者编号 | ||
created_at DATETIME, -- 创建时间 | ||
updated_at DATETIME, -- 更新时间 | ||
deleted_at DATETIME, -- 删除时间 | ||
PRIMARY KEY(id) | ||
); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
package db | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"strings" | ||
|
||
"github.com/gogf/gf/v2/container/gmap" | ||
"github.com/gogf/gf/v2/database/gdb" | ||
) | ||
|
||
// func getSQLiteSchemaByCli(tableName string, db gdb.DB) (string, error) { | ||
// dbConf := db.GetConfig() | ||
// cmd := exec.Command("sqlite3", dbConf.Name, fmt.Sprintf(".schema %s", tableName)) | ||
// glog.Info(context.TODO(), "sqlite3", dbConf.Name, fmt.Sprintf("'.schema %s'", tableName)) | ||
// output, err := cmd.CombinedOutput() | ||
// if err != nil { | ||
// return "", err | ||
// } | ||
// return strings.TrimSpace(string(output)), nil | ||
// } | ||
|
||
func getSQLiteSchemaBySql(ctx context.Context, tableName string, db gdb.DB) (string, error) { | ||
schemaRes, err := db.GetValue(ctx, fmt.Sprintf(`SELECT sql FROM sqlite_master WHERE type='table' AND name='%s';`, tableName)) | ||
if err != nil { | ||
return "", err | ||
} | ||
return schemaRes.String(), nil | ||
} | ||
|
||
func getSQliteTableComments(createTableSql string) (tableComment, tableName string) { | ||
// 按照换行符分割文本 | ||
lines := strings.Split(createTableSql, "\n") | ||
|
||
// 循环输出每一行 | ||
for _, line := range lines { | ||
// 检查 createTableSql 是否包含 comment | ||
if strings.Contains(line, "--") { | ||
if strings.Contains(line, "CREATE TABLE") { | ||
tableName = getLastWord(strings.Split(line, "(")[0]) | ||
tableComment = strings.Split(line, "--")[1] | ||
} | ||
} | ||
} | ||
return | ||
} | ||
|
||
func getSQliteFieldsComments(createTableSql string) (fieldCommentMap gmap.Map, tableComment, tableName string) { | ||
// 按照换行符分割文本 | ||
lines := strings.Split(createTableSql, "\n") | ||
|
||
// 循环输出每一行 | ||
for _, line := range lines { | ||
// 检查 createTableSql 是否包含 comment | ||
if strings.Contains(line, "--") { | ||
if strings.Contains(line, "CREATE TABLE") { | ||
tableName = getLastWord(strings.Split(line, "(")[0]) | ||
tableComment = strings.Split(line, "--")[1] | ||
} else { | ||
firstWord := getFirstWord(line) | ||
lastWord := getLastWord(line) | ||
fieldCommentMap.Set(firstWord, lastWord) | ||
} | ||
} | ||
} | ||
return | ||
} | ||
|
||
func transSqliteTablesComment(ctx context.Context, tableNames []string, db gdb.DB) (tables []*TableComment, err error) { | ||
schemaStr := "" | ||
eleIgnore := "sqlite_sequence" | ||
for _, v := range tableNames { | ||
if v != eleIgnore { | ||
schemaStr, err = getSQLiteSchemaBySql(ctx, v, db) | ||
if err != nil { | ||
return | ||
} | ||
comment, _ := getSQliteTableComments(schemaStr) | ||
tables = append(tables, &TableComment{ | ||
Name: v, | ||
Comment: comment, | ||
}) | ||
} | ||
} | ||
return | ||
} | ||
|
||
func fixSqliteFieldsComment(ctx context.Context, tableName string, db gdb.DB, fields map[string]*gdb.TableField) (map[string]*gdb.TableField, error) { | ||
// 记录: db.DoSelect 无法执行 .开头的命令 | ||
// schemaRes, err := db.DoSelect(ctx, dbConf.Link, fmt.Sprintf(`.schema %s`, d.QuoteWord(table))) | ||
// 记录: 查询 sqlite_master 不响应任何结果 | ||
// s, err := db.Query(ctx, `select * from sqlite_master WHERE name="%s";`, tableName) | ||
// schemaStr, err := getSQLiteSchemaBySql(tableName, db) | ||
schemaStr, err := getSQLiteSchemaBySql(ctx, tableName, db) | ||
if err != nil { | ||
return fields, err | ||
} | ||
comments, _, _ := getSQliteFieldsComments(schemaStr) | ||
for i := range fields { | ||
if comments.Contains(i) { | ||
fields[i].Comment = comments.Get(i).(string) | ||
} | ||
} | ||
return fields, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
package db | ||
|
||
import ( | ||
"strings" | ||
"unicode" | ||
) | ||
|
||
// 判断字符是否为字母、数字或下划线 | ||
func isWordChar(r rune) bool { | ||
return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' | ||
} | ||
|
||
// 获取一行文本中的第一个完整单词 | ||
func getFirstWord(text string) string { | ||
fields := strings.FieldsFunc(text, func(r rune) bool { return !isWordChar(r) }) | ||
if len(fields) > 0 { | ||
return fields[0] | ||
} | ||
return "" | ||
} | ||
|
||
// 获取一行文本中的最后一个完整单词 | ||
func getLastWord(text string) string { | ||
fields := strings.FieldsFunc(text, func(r rune) bool { return !isWordChar(r) }) | ||
if len(fields) > 0 { | ||
return fields[len(fields)-1] | ||
} | ||
return "" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters