Skip to content

Commit

Permalink
Merge pull request #12 from milaboratory/mzn-schema-change
Browse files Browse the repository at this point in the history 
Better token structure and metric names
  • Loading branch information
@Snyssfx
Snyssfx authored Dec 18, 2024
2 parents a0f8a8c + d9b1f21 commit ee0b27f
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 55 deletions.
5 changes: 5 additions & 0 deletions .changeset/clean-kiwis-wait.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@platforma-open/milaboratories.software-small-binaries.mnz-client": minor
---

Better token structure and metric names
4 changes: 2 additions & 2 deletions mnz-client/cmd/mnz-client/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func main() {
// define flags
flag.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0])
println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] <argName>:<type=file>:<filepath>:<specs:size,linesNum>")
println("MI_LICENSE=E-ABC mnz-client -productName test_product [more flags..] <argName>:<type=file>:<filepath>:<specs:size,lines,sha256>")
println("Only type 'file' now supported.")
println("Program may send multiple specs. Connect them with comma ','")
flag.PrintDefaults()
Expand Down Expand Up @@ -56,7 +56,7 @@ func main() {
log.Fatal("Missing mandatory argument: productName")
}
if license == "" || !licenseFound {
log.Fatal("Missing mandatory env variable, set your private license string: MI_LICENSE=E-ABC")
log.Fatal("Missing mandatory env variable, set your private license string: MI_LICENSE=E-ABC..")
}

// prepare call
Expand Down
27 changes: 14 additions & 13 deletions mnz-client/internal/mnz/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ import (
var FileArgType = ArgType{
Name: ArgTypeFile,
AvailableSpecs: map[string]interface{}{
"size": nil,
"linesNum": nil,
"hash": nil,
"size": nil,
"lines": nil,
"sha256": nil,
},
RequiredSpecs: []string{"hash"},
RequiredSpecs: []string{"sha256"},
}

func fileSpecs(path string, mNames []string) (map[string]any, error) {
specs := make(map[string]any)
spec := make(map[string]any)

for _, mn := range mNames {
switch mn {
Expand All @@ -33,28 +33,28 @@ func fileSpecs(path string, mNames []string) (map[string]any, error) {
if err != nil {
return nil, err
}
specs[mn] = sz
spec[mn] = sz

case "linesNum":
case "lines":
count, err := countLinesInZip(path)
if err != nil {
return nil, err
}
specs[mn] = count
spec[mn] = count

case "hash":
case "sha256":
hash, err := fileSha256(path)
if err != nil {
return nil, err
}
specs[mn] = hash
spec[mn] = hash

default:
return nil, fmt.Errorf("spec name '%s' is not available", mn)
}
}

return specs, nil
return spec, nil
}

func countLinesInZip(path string) (int64, error) {
Expand Down Expand Up @@ -95,7 +95,8 @@ func fileSha256(path string) (string, error) {
defer f.Close()

h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
buf := make([]byte, 1024*1024) // using large read buffer to increase throughput
if _, err := io.CopyBuffer(h, f, buf); err != nil {
return "", fmt.Errorf("failed to get SHA256 hash of file %s, error %w", path, err)
}

Expand Down Expand Up @@ -144,7 +145,7 @@ func countLines(path string) (int64, error) {
var lc int64
scanner := bufio.NewScanner(f)
// https://stackoverflow.com/questions/8757389/reading-a-file-line-by-line-in-go/16615559#comment41613175_16615559
const maxCapacity int = 4096 * 8 // increase buffer more in case of long lines
const maxCapacity int = 1024 * 1024 // increase buffer more in case of long lines
buf := make([]byte, maxCapacity)
scanner.Buffer(buf, maxCapacity)
for scanner.Scan() {
Expand Down
64 changes: 32 additions & 32 deletions mnz-client/internal/mnz/file_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,100 +13,100 @@ func Test_fileSpecs(t *testing.T) {
mNames []string
}
tests := []struct {
name string
args args
wantSpecs map[string]any
wantErr string
name string
args args
wantSpec map[string]any
wantErr string
}{
{
name: "testfile.zip good",
args: args{
path: "testfile.zip",
mNames: []string{"size", "hash", "linesNum"},
mNames: []string{"size", "sha256", "lines"},
},
wantSpecs: map[string]any{
"size": int64(185),
"hash": "971780534dd6e29baa46821d52accf486dad1968ea987a86311a0afce885602a",
"linesNum": int64(4),
wantSpec: map[string]any{
"size": int64(185),
"sha256": "971780534dd6e29baa46821d52accf486dad1968ea987a86311a0afce885602a",
"lines": int64(4),
},
wantErr: "",
},
{
name: "testfile.txt good",
args: args{
path: "testfile.txt",
mNames: []string{"size", "hash", "linesNum"},
mNames: []string{"size", "sha256", "lines"},
},
wantSpecs: map[string]any{
"size": int64(19),
"hash": "e190705c7bf0c0ada5f6e36fec833f44a0574678267bd86df7e815f3799ecbb1",
"linesNum": int64(4),
wantSpec: map[string]any{
"size": int64(19),
"sha256": "e190705c7bf0c0ada5f6e36fec833f44a0574678267bd86df7e815f3799ecbb1",
"lines": int64(4),
},
wantErr: "",
},
{
name: "mixcr.csv big",
args: args{
path: "mixcr.csv",
mNames: []string{"size", "hash", "linesNum"},
mNames: []string{"size", "sha256", "lines"},
},
wantSpecs: map[string]any{
"size": int64(42772),
"hash": "0d38a35c3f4a0e5e28c17785d114fd74b85596d14e06dc2d951101e6e0683072",
"linesNum": int64(981),
wantSpec: map[string]any{
"size": int64(42772),
"sha256": "0d38a35c3f4a0e5e28c17785d114fd74b85596d14e06dc2d951101e6e0683072",
"lines": int64(981),
},
wantErr: "",
},
{
name: "multi.zip two files",
args: args{
path: "multi.zip",
mNames: []string{"size", "hash", "linesNum"},
mNames: []string{"size", "sha256", "lines"},
},
wantSpecs: nil,
wantErr: "zip multi.zip contains more than one file",
wantSpec: nil,
wantErr: "zip multi.zip contains more than one file",
},
{
name: "unknown Spec",
args: args{
path: "multi.zip",
mNames: []string{"AAA"},
},
wantSpecs: nil,
wantErr: "spec name 'AAA' is not available",
wantSpec: nil,
wantErr: "spec name 'AAA' is not available",
},
{
name: "unknown file",
args: args{
path: "multi123.zip",
mNames: []string{"size"},
},
wantSpecs: nil,
wantErr: "no such file or directory",
wantSpec: nil,
wantErr: "no such file or directory",
},
{
name: "zipped dir",
args: args{
path: "dir.zip",
mNames: []string{"linesNum"},
mNames: []string{"lines"},
},
wantSpecs: nil,
wantErr: "zip dir.zip contains directory",
wantSpec: nil,
wantErr: "zip dir.zip contains directory",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotSpecs, err := fileSpecs(tt.args.path, tt.args.mNames)
gotSpec, err := fileSpecs(tt.args.path, tt.args.mNames)
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr, "incorrect error from fileSpecs()")
} else {
require.NoError(t, err, "unexpected error from fileSpecs()")
}

if !reflect.DeepEqual(gotSpecs, tt.wantSpecs) {
if !reflect.DeepEqual(gotSpec, tt.wantSpec) {
t.Errorf("fileSpecs() \n"+
"got = %v,\n"+
"want = %v", gotSpecs, tt.wantSpecs)
"want = %v", gotSpec, tt.wantSpec)
}
})
}
Expand Down
16 changes: 8 additions & 8 deletions mnz-client/internal/mnz/mnz.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,15 @@ const (
)

type ArgType struct {
Name ArgTypeName `json:"name"`
AvailableSpecs map[string]interface{} `json:"-"`
RequiredSpecs []string `json:"-"`
Name ArgTypeName
AvailableSpecs map[string]interface{}
RequiredSpecs []string
}

type Arg struct {
Name string `json:"-"`
ArgType ArgType `json:"argType"`
Specs map[string]any `json:"specs"`
Type ArgTypeName `json:"type"`
Name string `json:"-"`
Spec map[string]any `json:"spec"`
}

func getArgType(s string) (ArgType, error) {
Expand Down Expand Up @@ -68,7 +68,7 @@ func PrepareArgs(args []string) (map[string]Arg, error) {
for _, arg := range args {
splittedArgs := strings.Split(arg, ":")
if len(splittedArgs) < filepathN { // specs could be empty
return nil, errors.New("invalid argument, agrument format '<type>:<name>:<filepath>[:metrics]'")
return nil, errors.New("invalid argument, argument format '<type>:<name>:<filepath>[:metrics]'")
}

// type
Expand Down Expand Up @@ -104,7 +104,7 @@ func PrepareArgs(args []string) (map[string]Arg, error) {
}
//}

result[argName] = Arg{argName, argType, runSpecs}
result[argName] = Arg{Name: argName, Type: argType.Name, Spec: runSpecs}
}

return result, nil
Expand Down

0 comments on commit ee0b27f

Please sign in to comment.