From eaf1caddcdfa07354ea5fda9cec0f94c728f70d4 Mon Sep 17 00:00:00 2001 From: songhang Date: Wed, 17 Jul 2024 12:38:35 -0600 Subject: [PATCH] provide filter and override to code generator --- README.md | 4 +- cmd/generate/cmd_util.go | 99 +++++++++++++++++++ cmd/generate/cmd_util_test.go | 95 ++++++++++++++++++ cmd/generate/main.go | 16 ++- internal/generator/generator.go | 4 +- internal/generator/generator_util.go | 33 +++++-- internal/generator/generator_util_test.go | 84 +++++++++++----- internal/generator/testdata/expected/filter | 32 ++++++ .../testdata/expected/filterAndOverride | 30 ++++++ internal/generator/testdata/expected/override | 40 ++++++++ .../testdata/source_codes/filter/filter.c | 29 ++++++ .../filterAndOverride/filter_and_override.c | 29 ++++++ .../testdata/source_codes/override/override.c | 29 ++++++ 13 files changed, 484 insertions(+), 40 deletions(-) create mode 100644 cmd/generate/cmd_util.go create mode 100644 cmd/generate/cmd_util_test.go create mode 100644 internal/generator/testdata/expected/filter create mode 100644 internal/generator/testdata/expected/filterAndOverride create mode 100644 internal/generator/testdata/expected/override create mode 100644 internal/generator/testdata/source_codes/filter/filter.c create mode 100644 internal/generator/testdata/source_codes/filterAndOverride/filter_and_override.c create mode 100644 internal/generator/testdata/source_codes/override/override.c diff --git a/README.md b/README.md index 3f84bce2..d1d4051c 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,9 @@ func main() { ``` # Generate support for third-party modules -This is an example that takes the path of a third-party module source code to generate support for it. Assume the source code path of that module is `./src`. You can call `go run cmd/generate/main.go ./src`. The stdout will be like +This is a simple example that takes the path of a third-party module source code to generate support for it. For detailed usage of the tool, please run +`go run ./cmd/generate/ --help` +Assuming the source code path of that module is `./src`, you can call `go run ./cmd/generate/ --src-path=./src`. The output will be similar to: ```go /** diff --git a/cmd/generate/cmd_util.go b/cmd/generate/cmd_util.go new file mode 100644 index 00000000..f20304e4 --- /dev/null +++ b/cmd/generate/cmd_util.go @@ -0,0 +1,99 @@ +/** + * Copyright (c) F5, Inc. + * + * This source code is licensed under the Apache License, Version 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +package main + +import ( + "errors" + "fmt" + "strings" + + "github.com/nginxinc/nginx-go-crossplane/internal/generator" +) + +type filterFlag struct { + filter map[string]struct{} +} + +func (f *filterFlag) Set(value string) error { + if f.filter == nil { + f.filter = make(map[string]struct{}) + } + f.filter[value] = struct{}{} + return nil +} + +func (f *filterFlag) String() string { + return fmt.Sprintf("%v", *f) +} + +type overrideItem struct { + directive string + masks []generator.Mask +} + +func (item *overrideItem) UnmarshalText(text []byte) error { + rawOverride := string(text) + + // rawStr should follow the format: directive:bitmask00|bitmask01|...,bitmask10|bitmask11|... + directive, definition, found := strings.Cut(rawOverride, ":") + if !found { + return errors.New("colon not found") + } + directive = strings.TrimSpace(directive) + + item.directive = directive + if directive == "" { + return errors.New("directive name is empty") + } + + definition = strings.TrimSpace(definition) + if definition == "" { + return errors.New("directive definition is empty") + } + + for _, varNamesStr := range strings.Split(definition, ",") { + varNamesList := strings.Split(varNamesStr, "|") + varNamesNum := len(varNamesList) + directiveMask := make(generator.Mask, varNamesNum) + + for idx, varName := range varNamesList { + trimmedName := strings.TrimSpace(varName) + if trimmedName == "" { + return errors.New("one directive bitmask is empty, check if there are unnecessary |") + } + + directiveMask[idx] = trimmedName + } + item.masks = append(item.masks, directiveMask) + } + + return nil +} + +type override map[string][]generator.Mask + +func (ov *override) String() string { + if ov == nil { + return "nil" + } + return fmt.Sprintf("%v", *ov) +} + +func (ov *override) Set(value string) error { + if *ov == nil { + *ov = override{} + } + var item overrideItem + err := item.UnmarshalText([]byte(value)) + if err != nil { + return fmt.Errorf("invalid override %s:%w", value, err) + } + + (*ov)[item.directive] = item.masks + return nil +} diff --git a/cmd/generate/cmd_util_test.go b/cmd/generate/cmd_util_test.go new file mode 100644 index 00000000..a26dcd60 --- /dev/null +++ b/cmd/generate/cmd_util_test.go @@ -0,0 +1,95 @@ +/** + * Copyright (c) F5, Inc. + * + * This source code is licensed under the Apache License, Version 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +package main + +import ( + "testing" + + "github.com/nginxinc/nginx-go-crossplane/internal/generator" + "github.com/stretchr/testify/require" +) + +//nolint:funlen +func TestOverrideParser(t *testing.T) { + t.Parallel() + tests := []struct { + name string + input string + expected overrideItem + wantErr bool + }{ + { + name: "normalFormat_pass", + input: "location:ngxHTTPMainConf|ngxConfTake12,ngxStreamMainConf", + + expected: overrideItem{ + directive: "location", + masks: []generator.Mask{ + {"ngxHTTPMainConf", "ngxConfTake12"}, + {"ngxStreamMainConf"}, + }, + }, + wantErr: false, + }, + { + name: "withSpaces_pass", + input: "hash:ngxHTTPUpsConf | ngxConfTake12, ngxStreamUpsConf | ngxConfTake12", + expected: overrideItem{ + directive: "hash", + masks: []generator.Mask{ + {"ngxHTTPUpsConf", "ngxConfTake12"}, + {"ngxStreamUpsConf", "ngxConfTake12"}, + }, + }, + wantErr: false, + }, + { + name: "withoutColon_fail", + input: "hashngxHTTPUpsConf | ngxConfTake12,ngxStreamUpsConf | ngxConfTake12", + wantErr: true, + }, + { + name: "colonLeftsideEmpty_fail", + input: " :ngxHTTPUpsConf | ngxConfTake12,ngxStreamUpsConf | ngxConfTake12", + wantErr: true, + }, + { + name: "colonRightsideEmpty_fail", + input: "hash: ", + wantErr: true, + }, + { + name: "emptyBitmask_fail", + input: "hash: ngxHTTPUpsConf| ", + wantErr: true, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var got overrideItem + err := got.UnmarshalText([]byte(tc.input)) + + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + // If the testcase wants an error and there is an error, skip the output file validation. + // Output makes no sense when there is an error. + if err != nil { + return + } + + require.Equal(t, tc.expected, got) + }) + } +} diff --git a/cmd/generate/main.go b/cmd/generate/main.go index 8b416f4e..573f818b 100644 --- a/cmd/generate/main.go +++ b/cmd/generate/main.go @@ -18,10 +18,22 @@ import ( func main() { var ( sourceCodePath = flag.String("src-path", "", - "the path of source code your want to generate support from, it can be either a file or a directory. (required)") + "The path of source code your want to generate support from, it can be either a file or a directory. (required)") + filterflags filterFlag + directiveOverride override ) + flag.Var(&filterflags, "filter", + "A list of strings specifying the directives to exclude from the output. "+ + "An example is: -filter directive1 -filter directive2...(optional)") + flag.Var(&directiveOverride, "override", + "A list of strings, used to override the output. "+ + "It should follow the format:{directive:bitmask00|bitmask01...,bitmask10|bitmask11...}"+"\n"+ + "An example is -override=log_format:ngxHTTPMainConf|ngxConf2More,ngxStreamMainConf|ngxConf2More"+"\n"+ + `To use | and , in command line, you may need to enclose your input in quotes, i.e. -override="directive:mask1,mask2,...". (optional)`) + flag.Parse() - err := generator.Generate(*sourceCodePath, os.Stdout) + + err := generator.Generate(*sourceCodePath, os.Stdout, filterflags.filter, directiveOverride) if err != nil { log.Fatal(err) } diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 145c6b7a..fe4b2abb 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -15,6 +15,6 @@ import ( // extract all the directives definitions from the .c and .cpp files in // sourcePath and its subdirectories, then output the corresponding directive // masks map named "directives" and matchFunc named "Match" via writer. -func Generate(sourcePath string, writer io.Writer) error { - return genFromSrcCode(sourcePath, "directives", "Match", writer) +func Generate(sourcePath string, writer io.Writer, filter map[string]struct{}, override map[string][]Mask) error { + return genFromSrcCode(sourcePath, "directives", "Match", writer, filter, override) } diff --git a/internal/generator/generator_util.go b/internal/generator/generator_util.go index 48a415de..1ab0c14a 100644 --- a/internal/generator/generator_util.go +++ b/internal/generator/generator_util.go @@ -20,14 +20,14 @@ import ( "strings" ) -// A mask is a list of string, includes several variable names, +// A Mask is a list of string, includes several variable names, // which specify a behavior of a directive. // An example is []string{"ngxHTTPMainConf", "ngxConfFlag",}. // A directive can have several masks. -type mask []string +type Mask []string type supportFileTmplStruct struct { - Directive2Masks map[string][]mask + Directive2Masks map[string][]Mask MapVariableName string MatchFnName string } @@ -99,8 +99,8 @@ var ngxVarNameToGo = map[string]string{ } //nolint:nonamedreturns -func masksFromFile(path string) (directive2Masks map[string][]mask, err error) { - directive2Masks = make(map[string][]mask, 0) +func masksFromFile(path string) (directive2Masks map[string][]Mask, err error) { + directive2Masks = make(map[string][]Mask, 0) byteContent, err := os.ReadFile(path) if err != nil { return nil, err @@ -143,8 +143,8 @@ func masksFromFile(path string) (directive2Masks map[string][]mask, err error) { } //nolint:nonamedreturns -func getMasksFromPath(path string) (directive2Masks map[string][]mask, err error) { - directive2Masks = make(map[string][]mask, 0) +func getMasksFromPath(path string) (directive2Masks map[string][]Mask, err error) { + directive2Masks = make(map[string][]Mask, 0) err = filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -184,12 +184,29 @@ func getMasksFromPath(path string) (directive2Masks map[string][]mask, err error return directive2Masks, nil } -func genFromSrcCode(codePath string, mapVariableName string, matchFnName string, writer io.Writer) error { +func genFromSrcCode(codePath string, mapVariableName string, matchFnName string, writer io.Writer, + filter map[string]struct{}, override map[string][]Mask) error { directive2Masks, err := getMasksFromPath(codePath) if err != nil { return err } + if len(filter) > 0 { + for d := range directive2Masks { + if _, found := filter[d]; found { + delete(directive2Masks, d) + } + } + } + + if override != nil { + for d := range directive2Masks { + if newMasks, found := override[d]; found { + directive2Masks[d] = newMasks + } + } + } + err = supportFileTmpl.Execute(writer, supportFileTmplStruct{ Directive2Masks: directive2Masks, MapVariableName: mapVariableName, diff --git a/internal/generator/generator_util_test.go b/internal/generator/generator_util_test.go index ebdaad3e..9df9bc0e 100644 --- a/internal/generator/generator_util_test.go +++ b/internal/generator/generator_util_test.go @@ -72,50 +72,82 @@ func getExpectedFilePath(relativePath string) (string, error) { //nolint:funlen,gocognit func TestGenSupFromSrcCode(t *testing.T) { t.Parallel() - tests := []struct { - name string + tests := map[string]struct { relativePath string wantErr bool + filter map[string]struct{} + override map[string][]Mask }{ - { - name: "normalDirectiveDefinition_pass", + "normalDirectiveDefinition_pass": { relativePath: "normalDefinition", wantErr: false, }, - { - name: "unknownBitmask_fail", + "unknownBitmask_fail": { relativePath: "unknownBitmask", wantErr: true, }, - { - name: "noDirectivesDefinition_fail", + "noDirectivesDefinition_fail": { relativePath: "noDirectives", wantErr: true, }, // If one directive was defined in several files, we should keep all // of the bitmask definitions - { - name: "directiveRepeatDefine_pass", + "directiveRepeatDefine_pass": { relativePath: "repeatDefine", }, - // If there are comments in definition, we should delete them - { - name: "commentsInDefinition_pass", + // If there are comments in directive definition, we should ignore them + "commentsInDefinition_pass": { relativePath: "commentsInDefinition", }, - // If there are comments in definition, we should delete them - { - name: "genFromSingleFile_pass", + "genFromSingleFile_pass": { relativePath: "single_file.c", }, - { - name: "fullNgxBitmaskCover_pass", + "fullNgxBitmaskCover_pass": { relativePath: "fullNgxBitmaskCover", }, + "testFilter_pass": { + relativePath: "filter", + filter: map[string]struct{}{"my_directive_2": {}, "my_directive_3": {}}, + }, + "testOverride_pass": { + relativePath: "override", + override: map[string][]Mask{ + "my_directive_1": { + Mask{"ngxHTTPMainConf", "ngxConfTake1"}, + Mask{"ngxHTTPMainConf", "ngxConfTake2"}, + }, + "my_directive_3": { + Mask{"ngxHTTPMainConf", "ngxConfTake2"}, + Mask{"ngxHTTPMainConf", "ngxConfTake3"}, + }, + }, + }, + "testFilterAndOverride_pass": { + relativePath: "filterAndOverride", + filter: map[string]struct{}{ + "my_directive_1": {}, + "my_directive_2": {}, + "my_directive_3": {}, + }, + override: map[string][]Mask{ + "my_directive_1": { + Mask{"ngxHTTPMainConf", "ngxConfTake1"}, + Mask{"ngxHTTPMainConf", "ngxConfTake2"}, + }, + "my_directive_3": { + Mask{"ngxHTTPMainConf", "ngxConfTake2"}, + Mask{"ngxHTTPMainConf", "ngxConfTake3"}, + }, + "my_directive_4": { + Mask{"ngxHTTPMainConf", "ngxConfTake2"}, + Mask{"ngxHTTPMainConf", "ngxConfTake3"}, + }, + }, + }, } - for _, tc := range tests { + for name, tc := range tests { tc := tc - t.Run(tc.name, func(t *testing.T) { + t.Run(name, func(t *testing.T) { t.Parallel() var err error codePath, err := getTestSrcCodePath(tc.relativePath) @@ -125,14 +157,12 @@ func TestGenSupFromSrcCode(t *testing.T) { var buf bytes.Buffer - err = genFromSrcCode(codePath, "directives", "Match", &buf) - - if !tc.wantErr && err != nil { - t.Fatal(err) - } + err = genFromSrcCode(codePath, "directives", "Match", &buf, tc.filter, tc.override) - if tc.wantErr && err == nil { - t.Fatal("expected error, got nil") + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) } // If the testcase wants an error and there is an error, skip the output file validation, diff --git a/internal/generator/testdata/expected/filter b/internal/generator/testdata/expected/filter new file mode 100644 index 00000000..0288b684 --- /dev/null +++ b/internal/generator/testdata/expected/filter @@ -0,0 +1,32 @@ +/** + * Copyright (c) F5, Inc. + * + * This source code is licensed under the Apache License, Version 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Code generated by generator; DO NOT EDIT. +// All the definitions are extracted from the source code +// Each bit mask describes these behaviors: +// - how many arguments the directive can take +// - whether or not it is a block directive +// - whether this is a flag (takes one argument that's either "on" or "off") +// - which contexts it's allowed to be in + +package crossplane + +var directives = map[string][]uint{ + "my_directive_1": { + ngxHTTPMainConf | ngxConfTake2, + }, + "my_directive_4": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxConfTake4, + }, +} + +// Match is a matchFunc for parsing an NGINX config that contains the +// preceding directives. +func Match(directive string) ([]uint, bool) { + m, ok := directives[directive] + return m, ok +} \ No newline at end of file diff --git a/internal/generator/testdata/expected/filterAndOverride b/internal/generator/testdata/expected/filterAndOverride new file mode 100644 index 00000000..61d8e4ef --- /dev/null +++ b/internal/generator/testdata/expected/filterAndOverride @@ -0,0 +1,30 @@ +/** + * Copyright (c) F5, Inc. + * + * This source code is licensed under the Apache License, Version 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Code generated by generator; DO NOT EDIT. +// All the definitions are extracted from the source code +// Each bit mask describes these behaviors: +// - how many arguments the directive can take +// - whether or not it is a block directive +// - whether this is a flag (takes one argument that's either "on" or "off") +// - which contexts it's allowed to be in + +package crossplane + +var directives = map[string][]uint{ + "my_directive_4": { + ngxHTTPMainConf | ngxConfTake2, + ngxHTTPMainConf | ngxConfTake3, + }, +} + +// Match is a matchFunc for parsing an NGINX config that contains the +// preceding directives. +func Match(directive string) ([]uint, bool) { + m, ok := directives[directive] + return m, ok +} \ No newline at end of file diff --git a/internal/generator/testdata/expected/override b/internal/generator/testdata/expected/override new file mode 100644 index 00000000..652b2c55 --- /dev/null +++ b/internal/generator/testdata/expected/override @@ -0,0 +1,40 @@ +/** + * Copyright (c) F5, Inc. + * + * This source code is licensed under the Apache License, Version 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Code generated by generator; DO NOT EDIT. +// All the definitions are extracted from the source code +// Each bit mask describes these behaviors: +// - how many arguments the directive can take +// - whether or not it is a block directive +// - whether this is a flag (takes one argument that's either "on" or "off") +// - which contexts it's allowed to be in + +package crossplane + +var directives = map[string][]uint{ + "my_directive_1": { + ngxHTTPMainConf | ngxConfTake1, + ngxHTTPMainConf | ngxConfTake2, + }, + "my_directive_2": { + ngxHTTPMainConf | ngxConfFlag, + }, + "my_directive_3": { + ngxHTTPMainConf | ngxConfTake2, + ngxHTTPMainConf | ngxConfTake3, + }, + "my_directive_4": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxConfTake4, + }, +} + +// Match is a matchFunc for parsing an NGINX config that contains the +// preceding directives. +func Match(directive string) ([]uint, bool) { + m, ok := directives[directive] + return m, ok +} \ No newline at end of file diff --git a/internal/generator/testdata/source_codes/filter/filter.c b/internal/generator/testdata/source_codes/filter/filter.c new file mode 100644 index 00000000..02035bed --- /dev/null +++ b/internal/generator/testdata/source_codes/filter/filter.c @@ -0,0 +1,29 @@ +static ngx_command_t my_directives[] = { + + { ngx_string("my_directive_1"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_2"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_3"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_NOARGS, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_4"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE4, + 0, + 0, + 0, + NULL }, + + ngx_null_command +}; \ No newline at end of file diff --git a/internal/generator/testdata/source_codes/filterAndOverride/filter_and_override.c b/internal/generator/testdata/source_codes/filterAndOverride/filter_and_override.c new file mode 100644 index 00000000..02035bed --- /dev/null +++ b/internal/generator/testdata/source_codes/filterAndOverride/filter_and_override.c @@ -0,0 +1,29 @@ +static ngx_command_t my_directives[] = { + + { ngx_string("my_directive_1"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_2"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_3"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_NOARGS, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_4"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE4, + 0, + 0, + 0, + NULL }, + + ngx_null_command +}; \ No newline at end of file diff --git a/internal/generator/testdata/source_codes/override/override.c b/internal/generator/testdata/source_codes/override/override.c new file mode 100644 index 00000000..02035bed --- /dev/null +++ b/internal/generator/testdata/source_codes/override/override.c @@ -0,0 +1,29 @@ +static ngx_command_t my_directives[] = { + + { ngx_string("my_directive_1"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_2"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_3"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_NOARGS, + 0, + 0, + 0, + NULL }, + { ngx_string("my_directive_4"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE4, + 0, + 0, + 0, + NULL }, + + ngx_null_command +}; \ No newline at end of file