Move some codes to statement sub package

Move some codes to statement sub package

Revert change for delete

Refactor new engine

Fix tests

Improve quote policy

Names with upper charactor on postgres will need quotes

Fix bug

Add new quote parameter on tests

Fix tests

Fix quotes

Improve quote policy

Fix cache bug

Fix map with cols

Fix rows bug

Improve dialect interface

Return sqls for create table

Code improvement

Fix find alias bug

Improve insert

Only replace quotes when necessary

Move value2interface from session to statement package

Remove duplicated code

Fix mssql issue with duplicate columns.

Fix err，add postgres column ARRAY

Fix table name

Ignore schema when dbtype is not postgres

Support count with cols

Fix bug when dump

Fix batch insert interface slice be panic

Fix master/slave bug

Fix pk bug

Fix dump/import bug 

Fix setschema

Fix dump bug 

Oracle : Local Naming Method

Stop using github.com/xorm/core

Fix find and count bug

Move processor function into one file

Fix duplicated deleted condition on FindAndCount

Don't keep db on dialects

Add more tests for FindAndCount

Fix dump test

Fix postgres schema problem

Fix quote with blank

Fix lint errors

Move column string to standalone method

Support session id

Improve insert map generating SQL

Move all integrations tests to a standalone sub package

caches/cache.go

@@ -0,0 +1,99 @@
+// Copyright 2019 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package caches
+
+import (
+	"bytes"
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/xormplus/xorm/schemas"
+)
+
+const (
+	// CacheExpired is default cache expired time
+	CacheExpired = 60 * time.Minute
+	// CacheMaxMemory is not use now
+	CacheMaxMemory = 256
+	// CacheGcInterval represents interval time to clear all expired nodes
+	CacheGcInterval = 10 * time.Minute
+	// CacheGcMaxRemoved represents max nodes removed when gc
+	CacheGcMaxRemoved = 20
+)
+
+// list all the errors
+var (
+	ErrCacheMiss = errors.New("xorm/cache: key not found")
+	ErrNotStored = errors.New("xorm/cache: not stored")
+	// ErrNotExist record does not exist error
+	ErrNotExist = errors.New("Record does not exist")
+)
+
+// CacheStore is a interface to store cache
+type CacheStore interface {
+	// key is primary key or composite primary key
+	// value is struct's pointer
+	// key format : <tablename>-p-<pk1>-<pk2>...
+	Put(key string, value interface{}) error
+	Get(key string) (interface{}, error)
+	Del(key string) error
+}
+
+// Cacher is an interface to provide cache
+// id format : u-<pk1>-<pk2>...
+type Cacher interface {
+	GetIds(tableName, sql string) interface{}
+	GetBean(tableName string, id string) interface{}
+	PutIds(tableName, sql string, ids interface{})
+	PutBean(tableName string, id string, obj interface{})
+	DelIds(tableName, sql string)
+	DelBean(tableName string, id string)
+	ClearIds(tableName string)
+	ClearBeans(tableName string)
+}
+
+func encodeIds(ids []schemas.PK) (string, error) {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	err := enc.Encode(ids)
+
+	return buf.String(), err
+}
+
+func decodeIds(s string) ([]schemas.PK, error) {
+	pks := make([]schemas.PK, 0)
+
+	dec := gob.NewDecoder(strings.NewReader(s))
+	err := dec.Decode(&pks)
+
+	return pks, err
+}
+
+// GetCacheSql returns cacher PKs via SQL
+func GetCacheSql(m Cacher, tableName, sql string, args interface{}) ([]schemas.PK, error) {
+	bytes := m.GetIds(tableName, GenSqlKey(sql, args))
+	if bytes == nil {
+		return nil, errors.New("Not Exist")
+	}
+	return decodeIds(bytes.(string))
+}
+
+// PutCacheSql puts cacher SQL and PKs
+func PutCacheSql(m Cacher, ids []schemas.PK, tableName, sql string, args interface{}) error {
+	bytes, err := encodeIds(ids)
+	if err != nil {
+		return err
+	}
+	m.PutIds(tableName, GenSqlKey(sql, args), bytes)
+	return nil
+}
+
+// GenSqlKey generates cache key
+func GenSqlKey(sql string, args interface{}) string {
+	return fmt.Sprintf("%v-%v", sql, args)
+}

caches/encode.go

@@ -0,0 +1,58 @@
+// Copyright 2020 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package caches
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/gob"
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+// md5 hash string
+func Md5(str string) string {
+	m := md5.New()
+	io.WriteString(m, str)
+	return fmt.Sprintf("%x", m.Sum(nil))
+}
+func Encode(data interface{}) ([]byte, error) {
+	//return JsonEncode(data)
+	return GobEncode(data)
+}
+
+func Decode(data []byte, to interface{}) error {
+	//return JsonDecode(data, to)
+	return GobDecode(data, to)
+}
+
+func GobEncode(data interface{}) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(&data)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func GobDecode(data []byte, to interface{}) error {
+	buf := bytes.NewBuffer(data)
+	dec := gob.NewDecoder(buf)
+	return dec.Decode(to)
+}
+
+func JsonEncode(data interface{}) ([]byte, error) {
+	val, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+	return val, nil
+}
+
+func JsonDecode(data []byte, to interface{}) error {
+	return json.Unmarshal(data, to)
+}

caches/leveldb.go

@@ -0,0 +1,94 @@
+// Copyright 2020 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package caches
+
+import (
+	"log"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// LevelDBStore implements CacheStore provide local machine
+type LevelDBStore struct {
+	store *leveldb.DB
+	Debug bool
+	v     interface{}
+}
+
+var _ CacheStore = &LevelDBStore{}
+
+func NewLevelDBStore(dbfile string) (*LevelDBStore, error) {
+	db := &LevelDBStore{}
+	h, err := leveldb.OpenFile(dbfile, nil)
+	if err != nil {
+		return nil, err
+	}
+	db.store = h
+	return db, nil
+}
+
+func (s *LevelDBStore) Put(key string, value interface{}) error {
+	val, err := Encode(value)
+	if err != nil {
+		if s.Debug {
+			log.Println("[LevelDB]EncodeErr: ", err, "Key:", key)
+		}
+		return err
+	}
+	err = s.store.Put([]byte(key), val, nil)
+	if err != nil {
+		if s.Debug {
+			log.Println("[LevelDB]PutErr: ", err, "Key:", key)
+		}
+		return err
+	}
+	if s.Debug {
+		log.Println("[LevelDB]Put: ", key)
+	}
+	return err
+}
+
+func (s *LevelDBStore) Get(key string) (interface{}, error) {
+	data, err := s.store.Get([]byte(key), nil)
+	if err != nil {
+		if s.Debug {
+			log.Println("[LevelDB]GetErr: ", err, "Key:", key)
+		}
+		if err == leveldb.ErrNotFound {
+			return nil, ErrNotExist
+		}
+		return nil, err
+	}
+
+	err = Decode(data, &s.v)
+	if err != nil {
+		if s.Debug {
+			log.Println("[LevelDB]DecodeErr: ", err, "Key:", key)
+		}
+		return nil, err
+	}
+	if s.Debug {
+		log.Println("[LevelDB]Get: ", key, s.v)
+	}
+	return s.v, err
+}
+
+func (s *LevelDBStore) Del(key string) error {
+	err := s.store.Delete([]byte(key), nil)
+	if err != nil {
+		if s.Debug {
+			log.Println("[LevelDB]DelErr: ", err, "Key:", key)
+		}
+		return err
+	}
+	if s.Debug {
+		log.Println("[LevelDB]Del: ", key)
+	}
+	return err
+}
+
+func (s *LevelDBStore) Close() {
+	s.store.Close()
+}

caches/leveldb_test.go

@@ -0,0 +1,39 @@
+// Copyright 2020 The Xorm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package caches
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLevelDBStore(t *testing.T) {
+	store, err := NewLevelDBStore("./level.db")
+	assert.NoError(t, err)
+
+	var kvs = map[string]interface{}{
+		"a": "b",
+	}
+	for k, v := range kvs {
+		assert.NoError(t, store.Put(k, v))
+	}
+
+	for k, v := range kvs {
+		val, err := store.Get(k)
+		assert.NoError(t, err)
+		assert.EqualValues(t, v, val)
+	}
+
+	for k := range kvs {
+		err := store.Del(k)
+		assert.NoError(t, err)
+	}
+
+	for k := range kvs {
+		_, err := store.Get(k)
+		assert.EqualValues(t, ErrNotExist, err)
+	}
+}

cache_lru.go → caches/lru.go

@@ -2,15 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package xorm
+package caches
 
 import (
 	"container/list"
 	"fmt"
 	"sync"
 	"time"
-
-	"github.com/xormplus/core"
 )
 
 // LRUCacher implments cache object facilities
@@ -19,23 +17,23 @@ type LRUCacher struct {
 	sqlList        *list.List
 	idIndex        map[string]map[string]*list.Element
 	sqlIndex       map[string]map[string]*list.Element
-	store          core.CacheStore
+	store          CacheStore
 	mutex          sync.Mutex
 	MaxElementSize int
 	Expired        time.Duration
 	GcInterval     time.Duration
 }
 
 // NewLRUCacher creates a cacher
-func NewLRUCacher(store core.CacheStore, maxElementSize int) *LRUCacher {
+func NewLRUCacher(store CacheStore, maxElementSize int) *LRUCacher {
 	return NewLRUCacher2(store, 3600*time.Second, maxElementSize)
 }
 
 // NewLRUCacher2 creates a cache include different params
-func NewLRUCacher2(store core.CacheStore, expired time.Duration, maxElementSize int) *LRUCacher {
+func NewLRUCacher2(store CacheStore, expired time.Duration, maxElementSize int) *LRUCacher {
 	cacher := &LRUCacher{store: store, idList: list.New(),
 		sqlList: list.New(), Expired: expired,
-		GcInterval: core.CacheGcInterval, MaxElementSize: maxElementSize,
+		GcInterval: CacheGcInterval, MaxElementSize: maxElementSize,
 		sqlIndex: make(map[string]map[string]*list.Element),
 		idIndex:  make(map[string]map[string]*list.Element),
 	}
@@ -57,7 +55,7 @@ func (m *LRUCacher) GC() {
 	defer m.mutex.Unlock()
 	var removedNum int
 	for e := m.idList.Front(); e != nil; {
-		if removedNum <= core.CacheGcMaxRemoved &&
+		if removedNum <= CacheGcMaxRemoved &&
 			time.Now().Sub(e.Value.(*idNode).lastVisit) > m.Expired {
 			removedNum++
 			next := e.Next()
@@ -71,7 +69,7 @@ func (m *LRUCacher) GC() {
 
 	removedNum = 0
 	for e := m.sqlList.Front(); e != nil; {
-		if removedNum <= core.CacheGcMaxRemoved &&
+		if removedNum <= CacheGcMaxRemoved &&
 			time.Now().Sub(e.Value.(*sqlNode).lastVisit) > m.Expired {
 			removedNum++
 			next := e.Next()
@@ -268,11 +266,11 @@ type sqlNode struct {
 }
 
 func genSQLKey(sql string, args interface{}) string {
-	return fmt.Sprintf("%v-%v", sql, args)
+	return fmt.Sprintf("%s-%v", sql, args)
 }
 
 func genID(prefix string, id string) string {
-	return fmt.Sprintf("%v-%v", prefix, id)
+	return fmt.Sprintf("%s-%s", prefix, id)
 }
 
 func newIDNode(tbName string, id string) *idNode {

cache_lru_test.go → caches/lru_test.go

@@ -2,13 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package xorm
+package caches
 
 import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/xormplus/core"
+	"github.com/xormplus/xorm/schemas"
 )
 
 func TestLRUCache(t *testing.T) {
@@ -20,7 +20,7 @@ func TestLRUCache(t *testing.T) {
 	cacher := NewLRUCacher(store, 10000)
 
 	tableName := "cache_object1"
-	pks := []core.PK{
+	pks := []schemas.PK{
 		{1},
 		{2},
 	}