Better token structure and metric names

.changeset/clean-kiwis-wait.md

@@ -0,0 +1,5 @@
+---
+"@platforma-open/milaboratories.software-small-binaries.mnz-client": minor
+---
+
+Better token structure and metric names

mnz-client/cmd/mnz-client/root.go

@@ -15,7 +15,7 @@ func main() {
 	// define flags
 	flag.Usage = func() {
 		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0])
-		println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] <argName>:<type=file>:<filepath>:<specs:size,linesNum>")
+		println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] <argName>:<type=file>:<filepath>:<specs:size,lines,sha256>")
 		println("Only type 'file' now supported.")
 		println("Program may send multiple specs. Connect them with comma ','")
 		flag.PrintDefaults()
@@ -56,7 +56,7 @@ func main() {
 		log.Fatal("Missing mandatory argument: productName")
 	}
 	if license == "" || !licenseFound {
-		log.Fatal("Missing mandatory env variable, set your private license string: MI_LICENSE=E-ABC")
+		log.Fatal("Missing mandatory env variable, set your private license string: MI_LICENSE=E-ABC..")
 	}
 
 	// prepare call

mnz-client/internal/mnz/file.go

@@ -15,15 +15,15 @@ import (
 var FileArgType = ArgType{
 	Name: ArgTypeFile,
 	AvailableSpecs: map[string]interface{}{
-		"size":     nil,
-		"linesNum": nil,
-		"hash":     nil,
+		"size":   nil,
+		"lines":  nil,
+		"sha256": nil,
 	},
-	RequiredSpecs: []string{"hash"},
+	RequiredSpecs: []string{"sha256"},
 }
 
 func fileSpecs(path string, mNames []string) (map[string]any, error) {
-	specs := make(map[string]any)
+	spec := make(map[string]any)
 
 	for _, mn := range mNames {
 		switch mn {
@@ -33,28 +33,28 @@ func fileSpecs(path string, mNames []string) (map[string]any, error) {
 			if err != nil {
 				return nil, err
 			}
-			specs[mn] = sz
+			spec[mn] = sz
 
-		case "linesNum":
+		case "lines":
 			count, err := countLinesInZip(path)
 			if err != nil {
 				return nil, err
 			}
-			specs[mn] = count
+			spec[mn] = count
 
-		case "hash":
+		case "sha256":
 			hash, err := fileSha256(path)
 			if err != nil {
 				return nil, err
 			}
-			specs[mn] = hash
+			spec[mn] = hash
 
 		default:
 			return nil, fmt.Errorf("spec name '%s' is not available", mn)
 		}
 	}
 
-	return specs, nil
+	return spec, nil
 }
 
 func countLinesInZip(path string) (int64, error) {
@@ -95,7 +95,8 @@ func fileSha256(path string) (string, error) {
 	defer f.Close()
 
 	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
+	buf := make([]byte, 1024*1024) // using large read buffer to increase throughput
+	if _, err := io.CopyBuffer(h, f, buf); err != nil {
 		return "", fmt.Errorf("failed to get SHA256 hash of file %s, error %w", path, err)
 	}
 
@@ -144,7 +145,7 @@ func countLines(path string) (int64, error) {
 	var lc int64
 	scanner := bufio.NewScanner(f)
 	// https://stackoverflow.com/questions/8757389/reading-a-file-line-by-line-in-go/16615559#comment41613175_16615559
-	const maxCapacity int = 4096 * 8 // increase buffer more in case of long lines
+	const maxCapacity int = 1024 * 1024 // increase buffer more in case of long lines
 	buf := make([]byte, maxCapacity)
 	scanner.Buffer(buf, maxCapacity)
 	for scanner.Scan() {

mnz-client/internal/mnz/file_test.go

@@ -13,100 +13,100 @@ func Test_fileSpecs(t *testing.T) {
 		mNames []string
 	}
 	tests := []struct {
-		name      string
-		args      args
-		wantSpecs map[string]any
-		wantErr   string
+		name     string
+		args     args
+		wantSpec map[string]any
+		wantErr  string
 	}{
 		{
 			name: "testfile.zip good",
 			args: args{
 				path:   "testfile.zip",
-				mNames: []string{"size", "hash", "linesNum"},
+				mNames: []string{"size", "sha256", "lines"},
 			},
-			wantSpecs: map[string]any{
-				"size":     int64(185),
-				"hash":     "971780534dd6e29baa46821d52accf486dad1968ea987a86311a0afce885602a",
-				"linesNum": int64(4),
+			wantSpec: map[string]any{
+				"size":   int64(185),
+				"sha256": "971780534dd6e29baa46821d52accf486dad1968ea987a86311a0afce885602a",
+				"lines":  int64(4),
 			},
 			wantErr: "",
 		},
 		{
 			name: "testfile.txt good",
 			args: args{
 				path:   "testfile.txt",
-				mNames: []string{"size", "hash", "linesNum"},
+				mNames: []string{"size", "sha256", "lines"},
 			},
-			wantSpecs: map[string]any{
-				"size":     int64(19),
-				"hash":     "e190705c7bf0c0ada5f6e36fec833f44a0574678267bd86df7e815f3799ecbb1",
-				"linesNum": int64(4),
+			wantSpec: map[string]any{
+				"size":   int64(19),
+				"sha256": "e190705c7bf0c0ada5f6e36fec833f44a0574678267bd86df7e815f3799ecbb1",
+				"lines":  int64(4),
 			},
 			wantErr: "",
 		},
 		{
 			name: "mixcr.csv big",
 			args: args{
 				path:   "mixcr.csv",
-				mNames: []string{"size", "hash", "linesNum"},
+				mNames: []string{"size", "sha256", "lines"},
 			},
-			wantSpecs: map[string]any{
-				"size":     int64(42772),
-				"hash":     "0d38a35c3f4a0e5e28c17785d114fd74b85596d14e06dc2d951101e6e0683072",
-				"linesNum": int64(981),
+			wantSpec: map[string]any{
+				"size":   int64(42772),
+				"sha256": "0d38a35c3f4a0e5e28c17785d114fd74b85596d14e06dc2d951101e6e0683072",
+				"lines":  int64(981),
 			},
 			wantErr: "",
 		},
 		{
 			name: "multi.zip two files",
 			args: args{
 				path:   "multi.zip",
-				mNames: []string{"size", "hash", "linesNum"},
+				mNames: []string{"size", "sha256", "lines"},
 			},
-			wantSpecs: nil,
-			wantErr:   "zip multi.zip contains more than one file",
+			wantSpec: nil,
+			wantErr:  "zip multi.zip contains more than one file",
 		},
 		{
 			name: "unknown Spec",
 			args: args{
 				path:   "multi.zip",
 				mNames: []string{"AAA"},
 			},
-			wantSpecs: nil,
-			wantErr:   "spec name 'AAA' is not available",
+			wantSpec: nil,
+			wantErr:  "spec name 'AAA' is not available",
 		},
 		{
 			name: "unknown file",
 			args: args{
 				path:   "multi123.zip",
 				mNames: []string{"size"},
 			},
-			wantSpecs: nil,
-			wantErr:   "no such file or directory",
+			wantSpec: nil,
+			wantErr:  "no such file or directory",
 		},
 		{
 			name: "zipped dir",
 			args: args{
 				path:   "dir.zip",
-				mNames: []string{"linesNum"},
+				mNames: []string{"lines"},
 			},
-			wantSpecs: nil,
-			wantErr:   "zip dir.zip contains directory",
+			wantSpec: nil,
+			wantErr:  "zip dir.zip contains directory",
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotSpecs, err := fileSpecs(tt.args.path, tt.args.mNames)
+			gotSpec, err := fileSpecs(tt.args.path, tt.args.mNames)
 			if tt.wantErr != "" {
 				require.ErrorContains(t, err, tt.wantErr, "incorrect error from fileSpecs()")
 			} else {
 				require.NoError(t, err, "unexpected error from fileSpecs()")
 			}
 
-			if !reflect.DeepEqual(gotSpecs, tt.wantSpecs) {
+			if !reflect.DeepEqual(gotSpec, tt.wantSpec) {
 				t.Errorf("fileSpecs() \n"+
 					"got  = %v,\n"+
-					"want = %v", gotSpecs, tt.wantSpecs)
+					"want = %v", gotSpec, tt.wantSpec)
 			}
 		})
 	}

mnz-client/internal/mnz/mnz.go

@@ -19,15 +19,15 @@ const (
 )
 
 type ArgType struct {
-	Name           ArgTypeName            `json:"name"`
-	AvailableSpecs map[string]interface{} `json:"-"`
-	RequiredSpecs  []string               `json:"-"`
+	Name           ArgTypeName
+	AvailableSpecs map[string]interface{}
+	RequiredSpecs  []string
 }
 
 type Arg struct {
-	Name    string         `json:"-"`
-	ArgType ArgType        `json:"argType"`
-	Specs   map[string]any `json:"specs"`
+	Type ArgTypeName    `json:"type"`
+	Name string         `json:"-"`
+	Spec map[string]any `json:"spec"`
 }
 
 func getArgType(s string) (ArgType, error) {
@@ -68,7 +68,7 @@ func PrepareArgs(args []string) (map[string]Arg, error) {
 	for _, arg := range args {
 		splittedArgs := strings.Split(arg, ":")
 		if len(splittedArgs) < filepathN { // specs could be empty
-			return nil, errors.New("invalid argument, agrument format '<type>:<name>:<filepath>[:metrics]'")
+			return nil, errors.New("invalid argument, argument format '<type>:<name>:<filepath>[:metrics]'")
 		}
 
 		// type
@@ -104,7 +104,7 @@ func PrepareArgs(args []string) (map[string]Arg, error) {
 		}
 		//}
 
-		result[argName] = Arg{argName, argType, runSpecs}
+		result[argName] = Arg{Name: argName, Type: argType.Name, Spec: runSpecs}
 	}
 
 	return result, nil