Read full config with gnmi get (#143)

Why I did it
Need to read full config for SONiC

How I did it
Read CONFIG DB and convert to json.

How to verify it
Verified with unit test and end2end test.

gnmi_server/server_test.go

@@ -448,6 +448,96 @@ func loadConfigDB(t *testing.T, rclient *redis.Client, mpi map[string]interface{
 	}
 }
 
+func initFullConfigDb(t *testing.T, namespace string) {
+	rclient := getConfigDbClient(t, namespace)
+	defer rclient.Close()
+	rclient.FlushDB()
+
+	fileName := "../testdata/CONFIG_DHCP_SERVER.txt"
+	config, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	config_map := loadConfig(t, "", config)
+	loadConfigDB(t, rclient, config_map)
+}
+
+func initFullCountersDb(t *testing.T, namespace string) {
+	rclient := getRedisClient(t, namespace)
+	defer rclient.Close()
+	rclient.FlushDB()
+
+	fileName := "../testdata/COUNTERS_PORT_NAME_MAP.txt"
+	countersPortNameMapByte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	mpi_name_map := loadConfig(t, "COUNTERS_PORT_NAME_MAP", countersPortNameMapByte)
+	loadDB(t, rclient, mpi_name_map)
+
+	fileName = "../testdata/COUNTERS_QUEUE_NAME_MAP.txt"
+	countersQueueNameMapByte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	mpi_qname_map := loadConfig(t, "COUNTERS_QUEUE_NAME_MAP", countersQueueNameMapByte)
+	loadDB(t, rclient, mpi_qname_map)
+
+	fileName = "../testdata/COUNTERS:Ethernet68.txt"
+	countersEthernet68Byte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	// "Ethernet68": "oid:0x1000000000039",
+	mpi_counter := loadConfig(t, "COUNTERS:oid:0x1000000000039", countersEthernet68Byte)
+	loadDB(t, rclient, mpi_counter)
+
+	fileName = "../testdata/COUNTERS:Ethernet1.txt"
+	countersEthernet1Byte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	// "Ethernet1": "oid:0x1000000000003",
+	mpi_counter = loadConfig(t, "COUNTERS:oid:0x1000000000003", countersEthernet1Byte)
+	loadDB(t, rclient, mpi_counter)
+
+	// "Ethernet64:0": "oid:0x1500000000092a"  : queue counter, to work as data noise
+	fileName = "../testdata/COUNTERS:oid:0x1500000000092a.txt"
+	counters92aByte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000092a", counters92aByte)
+	loadDB(t, rclient, mpi_counter)
+
+	// "Ethernet68:1": "oid:0x1500000000091c"  : queue counter, for COUNTERS/Ethernet68/Queue vpath test
+	fileName = "../testdata/COUNTERS:oid:0x1500000000091c.txt"
+	countersEeth68_1Byte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091c", countersEeth68_1Byte)
+	loadDB(t, rclient, mpi_counter)
+
+	// "Ethernet68:3": "oid:0x1500000000091e"  : lossless queue counter, for COUNTERS/Ethernet68/Pfcwd vpath test
+	fileName = "../testdata/COUNTERS:oid:0x1500000000091e.txt"
+	countersEeth68_3Byte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091e", countersEeth68_3Byte)
+	loadDB(t, rclient, mpi_counter)
+
+	// "Ethernet68:4": "oid:0x1500000000091f"  : lossless queue counter, for COUNTERS/Ethernet68/Pfcwd vpath test
+	fileName = "../testdata/COUNTERS:oid:0x1500000000091f.txt"
+	countersEeth68_4Byte, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		t.Fatalf("read file %v err: %v", fileName, err)
+	}
+	mpi_counter = loadConfig(t, "COUNTERS:oid:0x1500000000091f", countersEeth68_4Byte)
+	loadDB(t, rclient, mpi_counter)
+}
+
 func prepareConfigDb(t *testing.T, namespace string) {
 	rclient := getConfigDbClient(t, namespace)
 	defer rclient.Close()
@@ -3706,6 +3796,8 @@ print('%s')
 	s := createServer(t, 8080)
 	go runServer(t, s)
 	defer s.s.Stop()
+	initFullConfigDb(t, sdcfg.GetDbDefaultNamespace())
+	initFullCountersDb(t, sdcfg.GetDbDefaultNamespace())
 
 	path, _ := os.Getwd()
 	path = filepath.Dir(path)
@@ -3788,6 +3880,8 @@ func TestMasterArbitration(t *testing.T) {
 	go runServer(t, s)
 	defer s.s.Stop()
 
+	prepareDbTranslib(t)
+
 	tlsConfig := &tls.Config{InsecureSkipVerify: true}
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))}
 

sonic_data_client/mixed_db_client.go

@@ -337,7 +337,10 @@ func (c *MixedDbClient) populateDbtablePath(path *gnmipb.Path, value *gnmipb.Typ
 
 	tblPath.dbNamespace = dbNamespace
 	tblPath.dbName = targetDbName
-	tblPath.tableName = stringSlice[1]
+	tblPath.tableName = ""
+	if len(stringSlice) > 1 {
+		tblPath.tableName = stringSlice[1]
+	}
 	tblPath.delimitor = separator
 	tblPath.operation = opRemove
 	tblPath.index = -1
@@ -375,6 +378,7 @@ func (c *MixedDbClient) populateDbtablePath(path *gnmipb.Path, value *gnmipb.Typ
 	// <4> DB Table Key Field
 	// <5> DB Table Key Field Index
 	switch len(stringSlice) {
+	case 1: // only db name provided
 	case 2: // only table name provided
 		if tblPath.operation == opRemove {
 			res, err := redisDb.Keys(tblPath.tableName + "*").Result()
@@ -430,7 +434,13 @@ func (c *MixedDbClient) makeJSON_redis(msi *map[string]interface{}, key *string,
 	// TODO: Use Yang model to identify leaf-list
 	if key == nil && op == nil {
 		for f, v := range mfv {
-			if strings.HasSuffix(f, "@") {
+			// There is NULL field in CONFIG DB, we need to remove NULL field from configuration
+			// user@sonic:~$ redis-cli -n 4 hgetall "DHCP_SERVER|192.0.0.29"
+			// 1) "NULL"
+			// 2) "NULL"
+			if f == "NULL" {
+				continue
+			} else if strings.HasSuffix(f, "@") {
 				k := strings.TrimSuffix(f, "@")
 				slice := strings.Split(v, ",")
 				(*msi)[k] = slice
@@ -443,7 +453,9 @@ func (c *MixedDbClient) makeJSON_redis(msi *map[string]interface{}, key *string,
 
 	fp := map[string]interface{}{}
 	for f, v := range mfv {
-		if strings.HasSuffix(f, "@") {
+		if f == "NULL" {
+			continue
+		} else if strings.HasSuffix(f, "@") {
 			k := strings.TrimSuffix(f, "@")
 			slice := strings.Split(v, ",")
 			fp[k] = slice
@@ -478,8 +490,21 @@ func (c *MixedDbClient) tableData2Msi(tblPath *tablePath, useKey bool, op *strin
 	var err error
 	var fv map[string]string
 
-	//Only table name provided
-	if tblPath.tableKey == "" {
+	if tblPath.tableName == "" {
+		// Did no provide table name
+		// Get all tables in the DB
+		// TODO: read all tables in COUNTERS_DB
+		if tblPath.dbName == "COUNTERS_DB" {
+			return fmt.Errorf("Can not read all tables in COUNTERS_DB")
+		}
+		pattern = "*" + tblPath.delimitor + "*"
+		dbkeys, err = redisDb.Keys(pattern).Result()
+		if err != nil {
+			log.V(2).Infof("redis Keys failed for %v, pattern %s", tblPath, pattern)
+			return fmt.Errorf("redis Keys failed for %v, pattern %s %v", tblPath, pattern, err)
+		}
+	} else if tblPath.tableKey == "" {
+		// Only table name provided
 		// tables in COUNTERS_DB other than COUNTERS table doesn't have keys
 		if tblPath.dbName == "COUNTERS_DB" && tblPath.tableName != "COUNTERS" {
 			pattern = tblPath.tableName
@@ -503,7 +528,24 @@ func (c *MixedDbClient) tableData2Msi(tblPath *tablePath, useKey bool, op *strin
 			return err
 		}
 
-		if (tblPath.tableKey != "" && !useKey) || tblPath.tableName == dbkey {
+		if (tblPath.tableName == "") {
+			// Split dbkey string into two parts
+			// First part is table name and second part is key in table
+			keys := strings.SplitN(dbkey, tblPath.delimitor, 2)
+			tableName := keys[0]
+			key := keys[1]
+			table_msi, ok := (*msi)[tableName].(*map[string]interface{})
+			if !ok {
+				tm := make(map[string]interface{})
+				table_msi = &tm
+				(*msi)[tableName] = table_msi
+			}
+			err = c.makeJSON_redis(table_msi, &key, op, fv)
+			if err != nil {
+				log.V(2).Infof("makeJSON err %s for fv %v", err, fv)
+				return err
+			}
+		} else if (tblPath.tableKey != "" && !useKey) || tblPath.tableName == dbkey {
 			if c.encoding == gnmipb.Encoding_JSON_IETF {
 				err = c.makeJSON_redis(msi, nil, op, fv)
 				if err != nil {

test/test_gnmi_configdb.py

@@ -354,3 +354,11 @@ def test_gnmi_get_checkpoint_negative_03(self):
         ret, _ = gnmi_get(get_list)
         assert ret != 0, 'Invalid path'
 
+    def test_gnmi_get_full_01(self):
+        get_list = ['/sonic-db:CONFIG_DB/']
+
+        ret, msg_list = gnmi_get(get_list)
+        assert ret == 0, 'Fail to get full config'
+        assert "NULL" not in msg_list[0], 'Invalid config'
+        # Config must be valid json
+        config = json.loads(msg_list[0])

test/test_gnmi_countersdb.py

@@ -0,0 +1,21 @@
+
+import os
+import time
+from utils import gnmi_set, gnmi_get, gnmi_dump
+
+import pytest
+
+
+class TestGNMICountersDb:
+
+    def test_gnmi_get_full_01(self):
+        get_list = ['/sonic-db:COUNTERS_DB/']
+
+        ret, msg_list = gnmi_get(get_list)
+        assert ret != 0, 'Does not support to read all table in COUNTERS_DB'
+
+    def test_gnmi_get_table_01(self):
+        get_list = ['/sonic-db:COUNTERS_DB/COUNTERS']
+
+        ret, msg_list = gnmi_get(get_list)
+        assert ret == 0, 'Fail to read COUNTERS table, ' + msg_list[0]

testdata/CONFIG_DHCP_SERVER.txt

@@ -0,0 +1,12 @@
+{
+    "DEVICE_METADATA|localhost": {
+        "bgp_asn": "65100",
+        "cloudtype":"Public"
+    },
+    "DHCP_SERVER|192.0.0.1": {
+        "NULL": "NULL"
+    },
+    "DHCP_SERVER|192.0.0.2": {
+        "NULL": "NULL"
+    }
+}