From 325625da1d8f1a6ea5ebc24b796d2b666b4dc5c8 Mon Sep 17 00:00:00 2001 From: Dmitry Bolotin Date: Wed, 18 Dec 2024 20:27:48 +0200 Subject: [PATCH 1/2] Better token structure and metric names --- mnz-client/cmd/mnz-client/root.go | 4 ++-- mnz-client/internal/mnz/file.go | 17 +++++++++-------- mnz-client/internal/mnz/mnz.go | 16 ++++++++-------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/mnz-client/cmd/mnz-client/root.go b/mnz-client/cmd/mnz-client/root.go index 2c6d8a5..b3c2307 100644 --- a/mnz-client/cmd/mnz-client/root.go +++ b/mnz-client/cmd/mnz-client/root.go @@ -15,7 +15,7 @@ func main() { // define flags flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0]) - println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] :::") + println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] :::") println("Only type 'file' now supported.") println("Program may send multiple specs. Connect them with comma ','") flag.PrintDefaults() @@ -56,7 +56,7 @@ func main() { log.Fatal("Missing mandatory argument: productName") } if license == "" || !licenseFound { - log.Fatal("Missing mandatory env variable, set your private license string: MI_LICENSE=E-ABC") + log.Fatal("Missing mandatory env variable, set your private license string: MI_LICENSE=E-ABC..") } // prepare call diff --git a/mnz-client/internal/mnz/file.go b/mnz-client/internal/mnz/file.go index 687cecc..6a47af3 100644 --- a/mnz-client/internal/mnz/file.go +++ b/mnz-client/internal/mnz/file.go @@ -15,11 +15,11 @@ import ( var FileArgType = ArgType{ Name: ArgTypeFile, AvailableSpecs: map[string]interface{}{ - "size": nil, - "linesNum": nil, - "hash": nil, + "size": nil, + "line_count": nil, + "hash_sha256": nil, }, - RequiredSpecs: []string{"hash"}, + RequiredSpecs: []string{"hash_sha256"}, } func fileSpecs(path string, mNames []string) (map[string]any, error) { @@ -35,14 +35,14 @@ func fileSpecs(path string, mNames []string) (map[string]any, error) { } specs[mn] = sz - case "linesNum": + case "line_count": count, err := countLinesInZip(path) if err != nil { return nil, err } specs[mn] = count - case "hash": + case "hash_sha256": hash, err := fileSha256(path) if err != nil { return nil, err @@ -95,7 +95,8 @@ func fileSha256(path string) (string, error) { defer f.Close() h := sha256.New() - if _, err := io.Copy(h, f); err != nil { + buf := make([]byte, 1024*1024) // using large read buffer to increase throughput + if _, err := io.CopyBuffer(h, f, buf); err != nil { return "", fmt.Errorf("failed to get SHA256 hash of file %s, error %w", path, err) } @@ -144,7 +145,7 @@ func countLines(path string) (int64, error) { var lc int64 scanner := bufio.NewScanner(f) // https://stackoverflow.com/questions/8757389/reading-a-file-line-by-line-in-go/16615559#comment41613175_16615559 - const maxCapacity int = 4096 * 8 // increase buffer more in case of long lines + const maxCapacity int = 1024 * 1024 // increase buffer more in case of long lines buf := make([]byte, maxCapacity) scanner.Buffer(buf, maxCapacity) for scanner.Scan() { diff --git a/mnz-client/internal/mnz/mnz.go b/mnz-client/internal/mnz/mnz.go index 1e96d37..9c06dd0 100644 --- a/mnz-client/internal/mnz/mnz.go +++ b/mnz-client/internal/mnz/mnz.go @@ -19,15 +19,15 @@ const ( ) type ArgType struct { - Name ArgTypeName `json:"name"` - AvailableSpecs map[string]interface{} `json:"-"` - RequiredSpecs []string `json:"-"` + Name ArgTypeName + AvailableSpecs map[string]interface{} + RequiredSpecs []string } type Arg struct { - Name string `json:"-"` - ArgType ArgType `json:"argType"` - Specs map[string]any `json:"specs"` + Type ArgTypeName `json:"type"` + Name string `json:"-"` + Spec map[string]any `json:"spec"` } func getArgType(s string) (ArgType, error) { @@ -68,7 +68,7 @@ func PrepareArgs(args []string) (map[string]Arg, error) { for _, arg := range args { splittedArgs := strings.Split(arg, ":") if len(splittedArgs) < filepathN { // specs could be empty - return nil, errors.New("invalid argument, agrument format '::[:metrics]'") + return nil, errors.New("invalid argument, argument format '::[:metrics]'") } // type @@ -104,7 +104,7 @@ func PrepareArgs(args []string) (map[string]Arg, error) { } //} - result[argName] = Arg{argName, argType, runSpecs} + result[argName] = Arg{Name: argName, Type: argType.Name, Spec: runSpecs} } return result, nil From d9b1f21ff986714cdca64d422c5ad4179652f15c Mon Sep 17 00:00:00 2001 From: Dmitry Bolotin Date: Wed, 18 Dec 2024 20:42:54 +0200 Subject: [PATCH 2/2] final renaming & changesets --- .changeset/clean-kiwis-wait.md | 5 +++ mnz-client/cmd/mnz-client/root.go | 2 +- mnz-client/internal/mnz/file.go | 22 +++++----- mnz-client/internal/mnz/file_test.go | 64 ++++++++++++++-------------- 4 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 .changeset/clean-kiwis-wait.md diff --git a/.changeset/clean-kiwis-wait.md b/.changeset/clean-kiwis-wait.md new file mode 100644 index 0000000..6b2a1b4 --- /dev/null +++ b/.changeset/clean-kiwis-wait.md @@ -0,0 +1,5 @@ +--- +"@platforma-open/milaboratories.software-small-binaries.mnz-client": minor +--- + +Better token structure and metric names diff --git a/mnz-client/cmd/mnz-client/root.go b/mnz-client/cmd/mnz-client/root.go index b3c2307..42b8d4c 100644 --- a/mnz-client/cmd/mnz-client/root.go +++ b/mnz-client/cmd/mnz-client/root.go @@ -15,7 +15,7 @@ func main() { // define flags flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0]) - println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] :::") + println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] :::") println("Only type 'file' now supported.") println("Program may send multiple specs. Connect them with comma ','") flag.PrintDefaults() diff --git a/mnz-client/internal/mnz/file.go b/mnz-client/internal/mnz/file.go index 6a47af3..a320bfe 100644 --- a/mnz-client/internal/mnz/file.go +++ b/mnz-client/internal/mnz/file.go @@ -15,15 +15,15 @@ import ( var FileArgType = ArgType{ Name: ArgTypeFile, AvailableSpecs: map[string]interface{}{ - "size": nil, - "line_count": nil, - "hash_sha256": nil, + "size": nil, + "lines": nil, + "sha256": nil, }, - RequiredSpecs: []string{"hash_sha256"}, + RequiredSpecs: []string{"sha256"}, } func fileSpecs(path string, mNames []string) (map[string]any, error) { - specs := make(map[string]any) + spec := make(map[string]any) for _, mn := range mNames { switch mn { @@ -33,28 +33,28 @@ func fileSpecs(path string, mNames []string) (map[string]any, error) { if err != nil { return nil, err } - specs[mn] = sz + spec[mn] = sz - case "line_count": + case "lines": count, err := countLinesInZip(path) if err != nil { return nil, err } - specs[mn] = count + spec[mn] = count - case "hash_sha256": + case "sha256": hash, err := fileSha256(path) if err != nil { return nil, err } - specs[mn] = hash + spec[mn] = hash default: return nil, fmt.Errorf("spec name '%s' is not available", mn) } } - return specs, nil + return spec, nil } func countLinesInZip(path string) (int64, error) { diff --git a/mnz-client/internal/mnz/file_test.go b/mnz-client/internal/mnz/file_test.go index e30ed63..d7d232b 100644 --- a/mnz-client/internal/mnz/file_test.go +++ b/mnz-client/internal/mnz/file_test.go @@ -13,21 +13,21 @@ func Test_fileSpecs(t *testing.T) { mNames []string } tests := []struct { - name string - args args - wantSpecs map[string]any - wantErr string + name string + args args + wantSpec map[string]any + wantErr string }{ { name: "testfile.zip good", args: args{ path: "testfile.zip", - mNames: []string{"size", "hash", "linesNum"}, + mNames: []string{"size", "sha256", "lines"}, }, - wantSpecs: map[string]any{ - "size": int64(185), - "hash": "971780534dd6e29baa46821d52accf486dad1968ea987a86311a0afce885602a", - "linesNum": int64(4), + wantSpec: map[string]any{ + "size": int64(185), + "sha256": "971780534dd6e29baa46821d52accf486dad1968ea987a86311a0afce885602a", + "lines": int64(4), }, wantErr: "", }, @@ -35,12 +35,12 @@ func Test_fileSpecs(t *testing.T) { name: "testfile.txt good", args: args{ path: "testfile.txt", - mNames: []string{"size", "hash", "linesNum"}, + mNames: []string{"size", "sha256", "lines"}, }, - wantSpecs: map[string]any{ - "size": int64(19), - "hash": "e190705c7bf0c0ada5f6e36fec833f44a0574678267bd86df7e815f3799ecbb1", - "linesNum": int64(4), + wantSpec: map[string]any{ + "size": int64(19), + "sha256": "e190705c7bf0c0ada5f6e36fec833f44a0574678267bd86df7e815f3799ecbb1", + "lines": int64(4), }, wantErr: "", }, @@ -48,12 +48,12 @@ func Test_fileSpecs(t *testing.T) { name: "mixcr.csv big", args: args{ path: "mixcr.csv", - mNames: []string{"size", "hash", "linesNum"}, + mNames: []string{"size", "sha256", "lines"}, }, - wantSpecs: map[string]any{ - "size": int64(42772), - "hash": "0d38a35c3f4a0e5e28c17785d114fd74b85596d14e06dc2d951101e6e0683072", - "linesNum": int64(981), + wantSpec: map[string]any{ + "size": int64(42772), + "sha256": "0d38a35c3f4a0e5e28c17785d114fd74b85596d14e06dc2d951101e6e0683072", + "lines": int64(981), }, wantErr: "", }, @@ -61,10 +61,10 @@ func Test_fileSpecs(t *testing.T) { name: "multi.zip two files", args: args{ path: "multi.zip", - mNames: []string{"size", "hash", "linesNum"}, + mNames: []string{"size", "sha256", "lines"}, }, - wantSpecs: nil, - wantErr: "zip multi.zip contains more than one file", + wantSpec: nil, + wantErr: "zip multi.zip contains more than one file", }, { name: "unknown Spec", @@ -72,8 +72,8 @@ func Test_fileSpecs(t *testing.T) { path: "multi.zip", mNames: []string{"AAA"}, }, - wantSpecs: nil, - wantErr: "spec name 'AAA' is not available", + wantSpec: nil, + wantErr: "spec name 'AAA' is not available", }, { name: "unknown file", @@ -81,32 +81,32 @@ func Test_fileSpecs(t *testing.T) { path: "multi123.zip", mNames: []string{"size"}, }, - wantSpecs: nil, - wantErr: "no such file or directory", + wantSpec: nil, + wantErr: "no such file or directory", }, { name: "zipped dir", args: args{ path: "dir.zip", - mNames: []string{"linesNum"}, + mNames: []string{"lines"}, }, - wantSpecs: nil, - wantErr: "zip dir.zip contains directory", + wantSpec: nil, + wantErr: "zip dir.zip contains directory", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotSpecs, err := fileSpecs(tt.args.path, tt.args.mNames) + gotSpec, err := fileSpecs(tt.args.path, tt.args.mNames) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr, "incorrect error from fileSpecs()") } else { require.NoError(t, err, "unexpected error from fileSpecs()") } - if !reflect.DeepEqual(gotSpecs, tt.wantSpecs) { + if !reflect.DeepEqual(gotSpec, tt.wantSpec) { t.Errorf("fileSpecs() \n"+ "got = %v,\n"+ - "want = %v", gotSpecs, tt.wantSpecs) + "want = %v", gotSpec, tt.wantSpec) } }) }