Merge branch 'unstable' into pika_cache

.github/workflows/codeql.yml

@@ -13,10 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
   schedule:
     - cron: '25 19 * * 6'
 

.github/workflows/codis.yml

@@ -5,9 +5,9 @@ name: Codis
 
 on:
   push:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
   pull_request:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
 
 jobs:
 

.github/workflows/operator.yml

@@ -5,9 +5,9 @@ name: Operator
 
 on:
   push:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
   pull_request:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
 
 jobs:
 

.github/workflows/pika.yml

@@ -2,9 +2,9 @@ name: Pika
 
 on:
   push:
-    branches: [ "unstable", "3.5.0" , "*"]
+    branches: [ "unstable", "3.5.1" , "*"]
   pull_request:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
 
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
@@ -172,7 +172,7 @@ jobs:
       - name: Configure CMake
         run: |
           export CC=/usr/local/opt/gcc@10/bin/gcc-10 
-          cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
+          cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
 
       - name: Build
         run: |

.github/workflows/publish_nightly_docker_image.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   push_pika_to_registry:
-    timeout-minutes: 600
+    timeout-minutes: 1440
     name: Push Pika Docker image to Docker Hub
     runs-on: ubuntu-latest
     steps:
@@ -34,6 +34,7 @@ jobs:
             type=schedule,prefix={{branch}},pattern={{date 'YYYYMMDD'}}
       
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: .
@@ -71,6 +72,7 @@ jobs:
             type=schedule,prefix={{branch}},pattern={{date 'YYYYMMDD'}}
       
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: ./codis

.github/workflows/publish_release_docker_image.yml

@@ -8,7 +8,7 @@ jobs:
   push_pika_to_registry:
     name: Push Pika Docker image to Docker Hub
     runs-on: ubuntu-latest
-    timeout-minutes: 600
+    timeout-minutes: 1440
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
@@ -32,6 +32,7 @@ jobs:
           images: pikadb/pika
 
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: .
@@ -67,6 +68,7 @@ jobs:
           images: pikadb/codis
 
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: ./codis

.github/workflows/tools_go.yml

@@ -2,11 +2,11 @@ name: Tools_go_build
 
 on:
   push:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
     paths:
       - 'tools/**'
   pull_request:
-    branches: [ "unstable", "3.5.0", "*" ]
+    branches: [ "unstable", "3.5.1", "*" ]
     paths:
       - 'tools/**'
 

CHANGELOG.MD

@@ -1,3 +1,48 @@
+# v3.5.1
+
+
+
+## New features
+
+- slow log Adds statistics on queue waiting time [#1997](https://github.com/OpenAtomFoundation/pika/pull/1997) @[wangshao1](https://github.com/wangshao1)
+
+- ReplicationID is used for primary/secondary replication [#1951](https://github.com/OpenAtomFoundation/pika/pull/1951) @[Mixficsol](https://github.com/Mixficsol)
+
+- WAL uses the disablewal command to support dynamic shutdown [#2015](https://github.com/OpenAtomFoundation/pika/pull/2015) @[Mixficsol](https://github.com/Mixficsol)
+
+- The number of threads flushed and the number of threads compaction dynamically adjust into one [#2014](flush线程数和compaction线程数动态调整，合二为一) @[Tianpingan](https://github.com/Tianpingan)
+
+- The RocksDB version was upgraded to v8.3.3 [#1999](https://github.com/OpenAtomFoundation/pika/issues/1999) @[dingxiaoshuai123](https://github.com/dingxiaoshuai123)
+
+- Added documentation for configuring Pika in Macos environment [#2003](https://github.com/OpenAtomFoundation/pika/pull/2003) @[klboke](https://github.com/klboke)
+
+- Added the ability to periodically print the length of the work queue to quickly locate problems when the queue is blocked [#1978](https://github.com/OpenAtomFoundation/pika/pull/1978) @[Tianpingan](https://github.com/Tianpingan)
+
+- Added an indicator to detect the entire cluster using a `pika_exporter` [#1953](https://github.com/OpenAtomFoundation/pika/pull/1953) @[chenbt-hz](https://github.com/chenbt-hz)
+
+- Realize automatic registration of Pika service on K8s environment, and automatically register at startup, so as to realize self-organization of cluster [#1931](https://github.com/OpenAtomFoundation/pika/pull/1931) @[machinly](https://github.com/machinly)
+
+
+## bugfix
+
+- Reduces unnecessary log printing by the exporter, reducing CPU utilization [#1945](https://github.com/OpenAtomFoundation/pika/pull/1945) @[Mixficsol](https://github.com/Mixficsol)
+
+- The sentinel mechanism has been upgraded to carry out logical deletion of primary nodes that fall offline [#1949](https://github.com/OpenAtomFoundation/pika/pull/1949) @[Mixficsol](https://github.com/Mixficsol)
+
+- Adjust the rate_limit parameter to fix the situation that RPS is 0 during pressure measurement [#2009](https://github.com/OpenAtomFoundation/pika/pull/2009) @[chejinge](https://github.com/chejinge)
+
+- Fixed the logical determination of empty path when traversing data files in INFODATA command [#1996](https://github.com/OpenAtomFoundation/pika/pull/1996) @[Mixficsol](https://github.com/Mixficsol)
+
+- Improved the description of some missing parts of the document [#1962](https://github.com/OpenAtomFoundation/pika/pull/1962) @[baerwang](https://github.com/baerwang)
+
+- Use make-j to improve build speed [#1933](https://github.com/OpenAtomFoundation/pika/pull/1933) @[xiezheng-XD](https://github.com/xiezheng-XD)
+
+- Fixed an issue where large burrs appeared on the Codis line [#2016](https://github.com/OpenAtomFoundation/pika/pull/2016) @[chejinge](https://github.com/chejinge)
+
+- Fixed an issue where tools could not be compiled in Macos [#2011](https://github.com/OpenAtomFoundation/pika/pull/2011) @[A2ureStone](https://github.com/A2ureStone)
+
+
+
 # v3.5.0
 
 ## Major features
@@ -26,7 +71,7 @@
 2. Support the INFO COMMANDSTATS command.
 3. Support the HELLO and SETNAME commands.
 4. Support the BLPOP and BRPOP commands.
-5. Added Pika original command DISKRECOVERY   
+5. Added Pika original command DISKRECOVERY
 
 ### 3.RocksDB Version Upgrade and Tiered Compression
 

CHANGELOG_CN.MD

@@ -1,3 +1,43 @@
+# v3.5.1
+
+## 新特性
+
+- slow log 增加队列等待时间统计 [#1997](https://github.com/OpenAtomFoundation/pika/pull/1997) @[wangshao1](https://github.com/wangshao1)
+
+- 主从复制使用 ReplicationID [#1951](https://github.com/OpenAtomFoundation/pika/pull/1951) @[Mixficsol](https://github.com/Mixficsol)
+
+- WAL 以 disablewal 命令方式支持动态关闭 [#2015](https://github.com/OpenAtomFoundation/pika/pull/2015) @[Mixficsol](https://github.com/Mixficsol)
+
+- flush 线程数和 compaction 线程数动态调整，合二为一 [#2014](flush线程数和compaction线程数动态调整，合二为一) @[Tianpingan](https://github.com/Tianpingan)
+
+- 升级了 RocksDB 版本到 v8.3.3 [#1999](https://github.com/OpenAtomFoundation/pika/issues/1999) @[dingxiaoshuai123](https://github.com/dingxiaoshuai123)
+
+- 增加了 Macos 环境下配置 Pika 的使用方法文档 [#2003](https://github.com/OpenAtomFoundation/pika/pull/2003) @[klboke](https://github.com/klboke)
+
+- 新增周期性打印工作队列的长度功能，在队列阻塞的时候可以快速定位问题 [#1978](https://github.com/OpenAtomFoundation/pika/pull/1978) @[Tianpingan](https://github.com/Tianpingan)
+
+- 新增利用一个 `pika_exporter` 检测整个集群的指标 [#1953](https://github.com/OpenAtomFoundation/pika/pull/1953) @[chenbt-hz](https://github.com/chenbt-hz)
+
+- 实现在 K8s 环境上 Pika 服务的自动注册，在启动时自动注册，从而实现集群的自组织 [#1931](https://github.com/OpenAtomFoundation/pika/pull/1931) @[machinly](https://github.com/machinly)
+
+## bugfix
+
+- 减少了 exporter 非必要的日志打印，降低 CPU 利用率 [#1945](https://github.com/OpenAtomFoundation/pika/pull/1945) @[Mixficsol](https://github.com/Mixficsol)
+
+- 升级了 sentinel 机制，对于掉线的主节点，进行逻辑删除 [#1949](https://github.com/OpenAtomFoundation/pika/pull/1949) @[Mixficsol](https://github.com/Mixficsol)
+
+- 调整 rate_limit 参数，修复压测时出现 RPS 为 0 的情况 [#2009](https://github.com/OpenAtomFoundation/pika/pull/2009) @[chejinge](https://github.com/chejinge)
+
+- 修复了 INFODATA 命令中对于遍历数据文件时出现空路径的逻辑判断 [#1996](https://github.com/OpenAtomFoundation/pika/pull/1996) @[Mixficsol](https://github.com/Mixficsol)
+
+- 完善了文档的一些部分缺失的描述 [#1962](https://github.com/OpenAtomFoundation/pika/pull/1962) @[baerwang](https://github.com/baerwang)
+
+- 使用 make -j 提高构建速度 [#1933](https://github.com/OpenAtomFoundation/pika/pull/1933) @[xiezheng-XD](https://github.com/xiezheng-XD)
+
+- 解决了 Codis 在线上出现大毛刺的问题 [#2016](https://github.com/OpenAtomFoundation/pika/pull/2016) @[chejinge](https://github.com/chejinge)
+
+- 修复了 Macos 环境下编译使用 tools 导致编译不过的问题 [#2011](https://github.com/OpenAtomFoundation/pika/pull/2011) @[A2ureStone](https://github.com/A2ureStone)
+
 # v3.5.0
 
 ## 重大特性
@@ -241,4 +281,4 @@
 * 将 /tools/codis2pika/test 中的 Redis 从 4.3.3 升级到 4.4.4 [#1536](https://github.com/OpenAtomFoundation/pika/pull/1536) [@dependabot]()
 * 将 golang.org/x/sys 从 /tools/codis2pika 中的 0.0.0-20210927094055-39ccf1dd6fa6 提升到 0.1.0 [#1535](https://github.com/OpenAtomFoundation/pika/pull/1535) [@dependabot]()
 * 用智能指针替换 new/delete [#1503](https://github.com/OpenAtomFoundation/pika/pull/1503)  [#1502](https://github.com/OpenAtomFoundation/pika/pull/1502) [#1493](https://github.com/OpenAtomFoundation/pika/pull/1493) [@cheniujh](https://github.com/cheniujh) [@A2ureStone](https://github.com/A2ureStone) [@iiiuwioajdks](https://github.com/iiiuwioajdks)
-* 用 glog 替换 fprintf [#1421](https://github.com/OpenAtomFoundation/pika/pull/1421) [@Mixficsol](https://github.com/Mixficsol)
+* 用 glog 替换 fprintf [#1421](https://github.com/OpenAtomFoundation/pika/pull/1421) [@Mixficsol](https://github.com/Mixficsol)

CMakeLists.txt

@@ -585,9 +585,9 @@ ExternalProject_Add(rocksdb
   ${LIBGPERF_NAME}
   ${LIBJEMALLOC_NAME}
   URL
-  https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz
+  https://github.com/facebook/rocksdb/archive/refs/tags/v8.3.3.tar.gz
   URL_HASH
-  MD5=3b4c97ee45df9c8a5517308d31ab008b
+  MD5=2815555a2ee2d12a7081a91fb988bf24
   DOWNLOAD_NO_PROGRESS
   1
   UPDATE_COMMAND
@@ -630,9 +630,9 @@ option(USE_PIKA_TOOLS "compile pika-tools" OFF)
 if (USE_PIKA_TOOLS)
   ExternalProject_Add(hiredis
     URL
-    https://github.com/vipshop/hiredis-vip/archive/refs/tags/0.3.0.tar.gz
+    https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz
     URL_HASH
-    MD5=e4d4a2928188b11f696d35ca1c52e9a4
+    MD5=119767d178cfa79718a80c83e0d0e849
     DOWNLOAD_NO_PROGRESS
     1
     UPDATE_COMMAND
@@ -654,7 +654,7 @@ if (USE_PIKA_TOOLS)
     INSTALL_COMMAND
     make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} install
   )
-  set(HIREDIS_LIBRARY ${INSTALL_LIBDIR}/libhiredis_vip.a)
+  set(HIREDIS_LIBRARY ${INSTALL_LIBDIR}/libhiredis.a)
 
   ExternalProject_Add(bz2
     URL

README.md

@@ -139,6 +139,10 @@ By default the compilation process is in 'release' mode. If you wanna debug this
   2. exec `rm -fr output` rebuild cmake (for complete recompilation)
 ```
 
+## Developing and Debugging with Pika
+
+- [Setting up the Development and Debugging Environment for Pika using CLion](./docs/ops/SetUpDevEnvironment_en.md)
+
 ## Dockerization
 
 ### Run with docker

README_CN.md

@@ -126,6 +126,10 @@ pika 默认使用`release`模式编译，不能调试，如果需要调试，需
   2. 执行 rm -fr output 重新生成cmkae（一般用于彻底重新编译）
 ```
 
+## pika 开发调试
+
+- [Pika 使用 CLion 搭建开发调试环境](./docs/ops/SetUpDevEnvironment.md)
+
 ## 容器化
 
 ### 使用docker运行

codis/pkg/proxy/backend.go

@@ -284,6 +284,7 @@ func (bc *BackendConn) loopReader(tasks <-chan *Request, c *redis.Conn, round in
 	}()
 	for r := range tasks {
 		resp, err := c.Decode()
+		r.ReceiveFromServerTime = time.Now().UnixNano()
 		if err != nil {
 			return bc.setResponse(r, nil, fmt.Errorf("backend conn failure, %s", err))
 		}
@@ -363,6 +364,7 @@ func (bc *BackendConn) loopWriter(round int) (err error) {
 		} else {
 			tasks <- r
 		}
+		r.SendToServerTime = time.Now().UnixNano()
 	}
 	return nil
 }

codis/pkg/proxy/request.go

@@ -21,8 +21,11 @@ type Request struct {
 	OpStr string
 	OpFlag
 
-	Database int32
-	UnixNano int64
+	Database              int32
+	ReceiveTime           int64
+	SendToServerTime      int64
+	ReceiveFromServerTime int64
+	TasksLen              int64
 
 	*redis.Resp
 	Err error
@@ -43,15 +46,15 @@ func (r *Request) MakeSubRequest(n int) []Request {
 		x.OpFlag = r.OpFlag
 		x.Broken = r.Broken
 		x.Database = r.Database
-		x.UnixNano = r.UnixNano
+		x.ReceiveTime = r.ReceiveTime
 	}
 	return sub
 }
 
 const GOLDEN_RATIO_PRIME_32 = 0x9e370001
 
 func (r *Request) Seed16() uint {
-	h32 := uint32(r.UnixNano) + uint32(uintptr(unsafe.Pointer(r)))
+	h32 := uint32(r.ReceiveTime) + uint32(uintptr(unsafe.Pointer(r)))
 	h32 *= GOLDEN_RATIO_PRIME_32
 	return uint(h32 >> 16)
 }

codis/pkg/proxy/session.go

@@ -169,7 +169,8 @@ func (s *Session) loopReader(tasks *RequestChan, d *Router) (err error) {
 		}
 		s.incrOpTotal()
 
-		if tasks.Buffered() > maxPipelineLen {
+		tasksLen := tasks.Buffered()
+		if tasksLen > maxPipelineLen {
 			return s.incrOpFails(nil, ErrTooManyPipelinedRequests)
 		}
 
@@ -181,7 +182,8 @@ func (s *Session) loopReader(tasks *RequestChan, d *Router) (err error) {
 		r.Multi = multi
 		r.Batch = &sync.WaitGroup{}
 		r.Database = s.database
-		r.UnixNano = start.UnixNano()
+		r.ReceiveTime = start.UnixNano()
+		r.TasksLen = int64(tasksLen)
 
 		if err := s.handleRequest(r, d); err != nil {
 			r.Resp = redis.NewErrorf("ERR handle request, %s", err)
@@ -235,6 +237,26 @@ func (s *Session) loopWriter(tasks *RequestChan) (err error) {
 		if fflush {
 			s.flushOpStats(false)
 		}
+		nowTime := time.Now().UnixNano()
+		duration := int64((nowTime - r.ReceiveTime) / 1e3)
+		if duration >= 50000 {
+			//client -> proxy -> server -> porxy -> client
+			//Record the waiting time from receiving the request from the client to sending it to the backend server
+			//the waiting time from sending the request to the backend server to receiving the response from the server
+			//the waiting time from receiving the server response to sending it to the client
+			var d0, d1, d2 int64 = -1, -1, -1
+			if r.SendToServerTime > 0 {
+				d0 = int64((r.SendToServerTime - r.ReceiveTime) / 1e3)
+			}
+			if r.SendToServerTime > 0 && r.ReceiveFromServerTime > 0 {
+				d1 = int64((r.ReceiveFromServerTime - r.SendToServerTime) / 1e3)
+			}
+			if r.ReceiveFromServerTime > 0 {
+				d2 = int64((nowTime - r.ReceiveFromServerTime) / 1e3)
+			}
+			log.Errorf("%s remote:%s, start_time(us):%d, duration(us): [%d, %d, %d], %d, tasksLen:%d",
+				time.Unix(r.ReceiveTime/1e9, 0).Format("2006-01-02 15:04:05"), s.Conn.RemoteAddr(), r.ReceiveTime/1e3, d0, d1, d2, duration, r.TasksLen)
+		}
 		return nil
 	})
 }
@@ -649,7 +671,7 @@ func (s *Session) getOpStats(opstr string) *opStats {
 func (s *Session) incrOpStats(r *Request, t redis.RespType) {
 	e := s.getOpStats(r.OpStr)
 	e.calls.Incr()
-	e.nsecs.Add(time.Now().UnixNano() - r.UnixNano)
+	e.nsecs.Add(time.Now().UnixNano() - r.ReceiveTime)
 	switch t {
 	case redis.TypeError:
 		e.redis.errors.Incr()

codis/pkg/topom/topom_sentinel.go

@@ -72,6 +72,7 @@ func (s *Topom) CheckAndSwitchSlavesAndMasters(filter func(index int, g *models.
 				if g.Servers[0].ReCallTimes >= s.Config().SentinelMasterDeadCheckTimes {
 					// Mark enters objective offline state
 					g.Servers[0].State = models.GroupServerStateOffline
+					g.Servers[0].ReplicaGroup = false
 				}
 				// Start the election master node
 				if g.Servers[0].State == models.GroupServerStateOffline {