Merge pull request #11 from feng9797/prod

Fault injection and Redis double-write
huaweicloud · Mar 18, 2022 · 4247afd · 4247afd
2 parents fc91806 + 48fd6b4
commit 4247afd
Show file tree

Hide file tree

Showing 87 changed files with 5,557 additions and 681 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,21 +1,25 @@
+#### 2022-03-18
+version: github.com/huaweicloud/devcloud-go v1.0.0  
+feature:
+1. package redis add "double-write" strategy mode.
+2. package mock add "Fault injection".
+3. package redis add "Fault injection service".
+4. package mysql add "Fault injection service".
+5. add web package, which is gin-gorm integration.
+
 #### 2022-01-04
 version: github.com/huaweicloud/devcloud-go v0.1.1  
 feature:
 1. mock package add etcd mock, replace test cases that rely on real etcd.
 
-#### 2021-12-27
-version: github.com/huaweicloud/devcloud-go v0.1.0  
-feature:
-1. add dms package, which is a high performance and high reliability kafka consumer.
-2. add mock package, which can mock redis, mysql and interface.
 #### 2021-12-25
 1. dms: persist the first N continuous offsets in offsetNode to the database and kafka broker, this will reduce repeated consumption of messages.
 2. change dms/method.go BizHandler from interface to function types.
 #### 2021-12-24
 1. add mock package, which contains interface mock, redis mock and mysql mock.
 
 #### 2021-12-16
-version: github.com/huaweicloud/devcloud-go v0.0.1  
+version: github.com/huaweicloud/devcloud-go v0.0.1
 feature:
 1. add dms which is a kafka consumer.
 

diff --git a/README.md b/README.md
@@ -1,8 +1,10 @@
 # Devcloud-go
 
 Devcloud-go provides a sql-driver for mysql, a redis client and a kafka consumer which named dms, you can use the driver and redis client with MAS or use them separately,
+at the same time, they also support use cases with injection faults for scenario simulation.
 The driver is developed based on [go-sql-driver/mysql v1.6.0](https://github.com/go-sql-driver/mysql), the redis client is developed based on [go-redis v8.11.3](https://github.com/go-redis/redis).
 The kafka consumer is developed based on [github.com/Shopify/sarama v1.29.1](https://github.com/Shopify/sarama).
+The mock package provides the simulation of MySQL, redis and etcd services, and realizes the fault injection function of MySQL and redis through TCP.
 This document introduces how to obtain and use Devcloud-go.
 
 ## Requirements
@@ -19,6 +21,7 @@ go get github.com/huaweicloud/devcloud-go
 * **sql-driver** : see details in [sql-driver/mysql/README.md](sql-driver/mysql/README.md)
 * **redis** : see details in [redis/README.md](redis/README.md)
 * **dms**: see details in [dms/README.md](dms/README.md)
+* **mock**: see details in [mock/README.md](mock/README.md)
 
 ## ChangeLog
 Detailed changes for each released version are documented in the [CHANGELOG.md](CHANGELOG.md).

diff --git a/common/etcd/client.go b/common/etcd/client.go
@@ -11,10 +11,12 @@
  * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations under the License.
  *
- * Package etcd defines EtcdClient interface, and use "go.etcd.io/etcd/client/v3"
- * implements the interface.
  */
 
+/*
+Package etcd defines EtcdClient interface, and use "go.etcd.io/etcd/client/v3"
+implements the interface.
+*/
 package etcd
 
 import (

diff --git a/common/etcd/mocks/EtcdClient.go b/common/etcd/mocks/EtcdClient.go
diff --git a/common/password/decipher.go b/common/password/decipher.go
@@ -15,6 +15,10 @@
  * user can set customize decipher by SetDecipher function.
  */
 
+/*
+Package password defines Decipher interface, which is used to decode password,
+user can set customize decipher by SetDecipher function.
+*/
 package password
 
 import "sync"

diff --git a/common/util/util.go b/common/util/util.go
@@ -11,9 +11,9 @@
  * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations under the License.
  *
- * Package util provides some util function, such as ValidateHostPort.
  */
 
+// Package util provides some util function, such as ValidateHostPort.
 package util
 
 import (

diff --git a/dms/consumer.go b/dms/consumer.go
@@ -11,10 +11,12 @@
  * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations under the License.
  *
- * Package dms implements a kafka consumer based on sarama, user can consume messages
- * asynchronous or synchronous with dms, and ensure message not lost.
  */
 
+/*
+Package dms implements a kafka consumer based on sarama, user can consume messages
+asynchronous or synchronous with dms, and ensure message not lost.
+*/
 package dms
 
 import (

diff --git a/dms/example/example.go b/dms/example/example.go
@@ -11,9 +11,9 @@
  * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations under the License.
  *
- * Package example provides an example for user how to use dms.
  */
 
+// Package example provides an example for user how to use dms.
 package example
 
 import (

diff --git a/dms/offset_manager.go b/dms/offset_manager.go
@@ -115,7 +115,7 @@ func (m *OffsetManager) handleOffsetOnCleanUp(offsetPersist OffsetPersist) int64
 	m.lock.RUnlock()
 	offset := minKey.(int64) + m.startOffset + int64(minNode.(*OffsetNode).maxContinuous())
 	if err := offsetPersist.Save(m.groupId, m.topic, m.partition, offset); err != nil {
-		log.Printf("WARNING: groupId/topic/partition %s/%s/%s persist %d fail on clean up, %v",
+		log.Printf("WARNING: groupId/topic/partition %s/%s/%d persist %d fail on clean up, %v",
 			m.groupId, m.topic, m.partition, offset, err)
 	}
 	return offset

diff --git a/go.mod b/go.mod
@@ -6,19 +6,26 @@ require (
 	github.com/RoaringBitmap/roaring v0.9.4
 	github.com/Shopify/sarama v1.29.1
 	github.com/alicebob/miniredis/v2 v2.15.1
+	github.com/astaxie/beego v1.12.3
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/dolthub/go-mysql-server v0.11.0
+	github.com/dolthub/vitess v0.0.0-20211013185428-a8845fb919c1
 	github.com/emirpasic/gods v1.12.0
+	github.com/gin-gonic/gin v1.7.7
 	github.com/go-redis/redis/v8 v8.11.3
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.15.0
-	github.com/panjf2000/ants/v2 v2.4.7
+	github.com/panjf2000/ants/v2 v2.4.6
 	github.com/stretchr/testify v1.7.0
+	github.com/tidwall/redcon v1.4.4
 	go.etcd.io/etcd/api/v3 v3.5.1
 	go.etcd.io/etcd/client/v3 v3.5.1
 	go.etcd.io/etcd/server/v3 v3.5.1
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	gopkg.in/fatih/pool.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gorm.io/driver/mysql v1.3.2
+	gorm.io/gorm v1.23.1
 )
diff --git a/img/mysql-configuration.png b/img/mysql-configuration.png
diff --git a/img/mysql-local-read-single-write.png b/img/mysql-local-read-single-write.png
diff --git a/img/mysql-single-read-write.png b/img/mysql-single-read-write.png
diff --git a/img/proxy.png b/img/proxy.png
diff --git a/img/redis-configuration.png b/img/redis-configuration.png
diff --git a/img/redis-double-write.png b/img/redis-double-write.png
diff --git a/img/redis-local-read-single-write.png b/img/redis-local-read-single-write.png
diff --git a/img/redis-single-read-write.png b/img/redis-single-read-write.png
diff --git a/mas/config.go b/mas/config.go
@@ -11,23 +11,22 @@
  * CONDITIONS OF ANY KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations under the License.
  *
- * Package mas contains mas properties configuration.
  */
 
+// Package mas contains mas properties configuration.
 package mas
 
 import "fmt"
 
 // PropertiesConfiguration yaml properties configuration entity
 type PropertiesConfiguration struct {
-	Version           string `yaml:"version"`
-	AppID             string `yaml:"appId"`
-	MonitorID         string `yaml:"monitorId"`
-	DatabaseName      string `yaml:"databaseName"`
-	DecipherClassName string `yaml:"decipherClassName"`
-	Cloud             string `yaml:"cloud"`
-	Region            string `yaml:"region"`
-	Azs               string `yaml:"azs"`
+	Version      string `yaml:"version"`
+	AppID        string `yaml:"appId"`
+	MonitorID    string `yaml:"monitorId"`
+	DatabaseName string `yaml:"databaseName"`
+	Cloud        string `yaml:"cloud"`
+	Region       string `yaml:"region"`
+	Azs          string `yaml:"azs"`
 }
 
 const propertiesConfigurationHashCodeFmt = "%s_%s_%s_%s"

diff --git a/mas/config_test.go b/mas/config_test.go
@@ -44,12 +44,11 @@ func TestPropertiesConfiguration_CalHashCode(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			p := &PropertiesConfiguration{
-				Version:           tt.fields.Version,
-				AppID:             tt.fields.AppID,
-				MonitorID:         tt.fields.MonitorID,
-				DatabaseName:      tt.fields.DatabaseName,
-				DecipherClassName: tt.fields.DecipherClassName,
-				Region:            tt.fields.Region,
+				Version:      tt.fields.Version,
+				AppID:        tt.fields.AppID,
+				MonitorID:    tt.fields.MonitorID,
+				DatabaseName: tt.fields.DatabaseName,
+				Region:       tt.fields.Region,
 			}
 			if got := p.CalHashCode(); got != tt.want {
 				t.Errorf("CalHashCode() = %v, want %v", got, tt.want)