From ac07f5ec4ceac0b24f14b751100ca9ae9753ff8c Mon Sep 17 00:00:00 2001 From: Ochi Daiki Date: Tue, 25 Jul 2023 01:26:22 +0900 Subject: [PATCH] Verify (#204) * WIP verify * update * wip * update * update * update * update --- .gitignore | 3 +- README.md | 62 ++++++++++++++++- cmd/gtree/main.go | 53 +++++++++++++++ cmd/gtree/verify.go | 11 +++ config.go | 23 ++++++- example/noexist/xxx | 0 pipeline_tree.go | 44 ++++++++++++ pipeline_tree_verifier.go | 45 +++++++++++++ simple_tree.go | 38 +++++++++++ simple_tree_verifier.go | 127 +++++++++++++++++++++++++++++++++++ testdata/sample10.md | 18 +++++ testdata/sample9.md | 13 ++++ tree.go | 2 + tree_handler.go | 11 +++ tree_handler_programmably.go | 20 ++++++ 15 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 cmd/gtree/verify.go create mode 100644 example/noexist/xxx create mode 100644 pipeline_tree_verifier.go create mode 100644 simple_tree_verifier.go create mode 100644 testdata/sample10.md create mode 100644 testdata/sample9.md diff --git a/.gitignore b/.gitignore index 220e1afb..93a7c6e0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,4 @@ root_h/ root_i/ root_j/ Primate/ -cmd/gtree/gtree -gtree \ No newline at end of file +cmd/gtree/gtree \ No newline at end of file diff --git a/README.md b/README.md index 40e7eb96..8885c662 100644 --- a/README.md +++ b/README.md @@ -107,13 +107,15 @@ USAGE: gtree [global options] command [command options] [arguments...] VERSION: - 1.8.7 / revision cdef0bf + 1.9.1 / revision 91a2226 COMMANDS: output, o, out Outputs tree from markdown. Let's try 'gtree template | gtree output'. mkdir, m Makes directories and files from markdown. It is possible to dry run. Let's try 'gtree template | gtree mkdir -e .go -e .md -e Makefile'. + verify, vf Verifies tree structure represented in markdown by comparing it with existing directories. + Let's try 'gtree template | gtree verify'. template, t, tmpl Outputs markdown template. Use it to try out gtree CLI. web, w, www Opens "Tree Maker" in your browser and shows the URL in terminal. gocode, gc, code Outputs a sample Go program calling "gtree" package. @@ -487,6 +489,55 @@ EOS invalid node name: /root ``` +### *Verify* subcommand +```console +$ gtree verify --help +NAME: + gtree verify - Verifies tree structure represented in markdown by comparing it with existing directories. + Let's try 'gtree template | gtree verify'. + +USAGE: + gtree verify [command options] [arguments...] + +OPTIONS: + --file value, -f value specify the path to markdown file. (default: stdin) + --two-spaces, --ts set this option when the markdown indent is 2 spaces. (default: tab spaces) + --four-spaces, --fs set this option when the markdown indent is 4 spaces. (default: tab spaces) + --target-dir value set this option if you want to specify the directory you want to verify. (default: current directory) + --strict set this option if you want strict directory match validation. (default: non strict) + --help, -h show help +``` + +#### Try it! + +```console +~/github.com/ddddddO/gtree +$ cat testdata/sample9.md +- example + - find_pipe_programmable-gtree + - main.go + - go-list_pipe_programmable-gtree + - main.go + - like_cli + - adapter + - executor.go + - indentation.go + - main.go + - kkk + - programmable + - main.go +~/github.com/ddddddO/gtree +$ cat testdata/sample9.md | gtree verify --strict +2023/07/25 01:04:04 +Extra paths exist: + example/noexist + example/noexist/xxx +Required paths does not exist: + example/like_cli/kkk +exit status 1 +``` + + # Package(1) / like CLI ## Installation @@ -598,6 +649,12 @@ func main() { You can use `gtree.WithFileExtensions` func to make specified extensions as file. +### *Verify* func + +#### `gtree.Verify` func verifies directories. + +You can use `gtree.WithTargetDir` func / `gtree.WithStrictVerify` func. + # Package(2) / generate a tree programmatically ## Installation @@ -922,6 +979,9 @@ func main() { } ``` +### *VerifyProgrammably* func + +You can use `gtree.WithTargetDir` func / `gtree.WithStrictVerify` func. # Process diff --git a/cmd/gtree/main.go b/cmd/gtree/main.go index b96c2c2a..4550eb09 100644 --- a/cmd/gtree/main.go +++ b/cmd/gtree/main.go @@ -98,6 +98,19 @@ func main() { }, } + verifyFlags := []cli.Flag{ + &cli.StringFlag{ + Name: "target-dir", + Usage: "set this option if you want to specify the directory you want to verify.", + DefaultText: "current directory", + }, + &cli.BoolFlag{ + Name: "strict", + Usage: "set this option if you want strict directory match validation.", + DefaultText: "non strict", + }, + } + templateFlags := []cli.Flag{ &cli.BoolFlag{ Name: "description", @@ -146,6 +159,15 @@ func main() { Before: notExistArgs, Action: actionMkdir, }, + { + Name: "verify", + Aliases: []string{"vf"}, + Usage: "Verifies tree structure represented in markdown by comparing it with existing directories.\n" + + "Let's try 'gtree template | gtree verify'.", + Flags: append(commonFlags, verifyFlags...), + Before: notExistArgs, + Action: actionVerify, + }, { Name: "template", Aliases: []string{"t", "tmpl"}, @@ -283,6 +305,37 @@ func isInputStdin(path string) bool { return path == "" || path == "-" } +func actionVerify(c *cli.Context) error { + var ( + in = os.Stdin + err error + ) + if !isInputStdin(c.Path("file")) { + in, err = os.Open(c.Path("file")) + if err != nil { + return exitErrOpen(err) + } + defer in.Close() + } + + oi, err := optionIndentation(c) + if err != nil { + return exitErrOpts(err) + } + options := []gtree.Option{oi, gtree.WithTargetDir(c.String("target-dir"))} + if c.Bool("strict") { + options = append(options, gtree.WithStrictVerify()) + } + + if err := verify(in, options); err != nil { + if errors.As(err, >ree.VerifyError{}) { + return errors.New(fmt.Sprintf("\n%s", err.Error())) + } + return err + } + return nil +} + func actionTemplate(c *cli.Context) error { if c.Bool("description") { return description.println() diff --git a/cmd/gtree/verify.go b/cmd/gtree/verify.go new file mode 100644 index 00000000..8f65c125 --- /dev/null +++ b/cmd/gtree/verify.go @@ -0,0 +1,11 @@ +package main + +import ( + "io" + + "github.com/ddddddO/gtree" +) + +func verify(in io.Reader, options []gtree.Option) error { + return gtree.Verify(in, options...) +} diff --git a/config.go b/config.go index 4dafef24..05f89d3f 100644 --- a/config.go +++ b/config.go @@ -12,6 +12,8 @@ type config struct { encode encode dryrun bool fileExtensions []string + targetDir string + strictVerify bool } func newConfig(options []Option) *config { @@ -24,9 +26,10 @@ func newConfig(options []Option) *config { directly: "├──", indirectly: "│ ", }, - space: spacesTab, - massive: false, - encode: encodeDefault, + space: spacesTab, + massive: false, + encode: encodeDefault, + targetDir: ".", } for _, opt := range options { if opt == nil { @@ -116,3 +119,17 @@ func WithFileExtensions(extensions []string) Option { c.fileExtensions = extensions } } + +// WithTargetDir returns function for specifing directory. Default is current directory. +func WithTargetDir(dir string) Option { + return func(c *config) { + c.targetDir = dir + } +} + +// WithStrictVerify returns function for verifing directory strictly. +func WithStrictVerify() Option { + return func(c *config) { + c.strictVerify = true + } +} diff --git a/example/noexist/xxx b/example/noexist/xxx new file mode 100644 index 00000000..e69de29b diff --git a/pipeline_tree.go b/pipeline_tree.go index 871c4426..bcd33952 100644 --- a/pipeline_tree.go +++ b/pipeline_tree.go @@ -14,6 +14,7 @@ type treePipeline struct { grower growerPipeline spreader spreaderPipeline mkdirer mkdirerPipeline + verifier verifierPipeline } var _ tree = (*treePipeline)(nil) @@ -37,6 +38,10 @@ func newTreePipeline(cfg *config) tree { return newMkdirerPipeline(fileExtensions) } + verifierFactory := func(targetDir string, strict bool) verifierPipeline { + return newVerifierPipeline(targetDir, strict) + } + return &treePipeline{ grower: growerFactory( cfg.lastNodeFormat, @@ -52,6 +57,10 @@ func newTreePipeline(cfg *config) tree { mkdirer: mkdirerFactory( cfg.fileExtensions, ), + verifier: verifierFactory( + cfg.targetDir, + cfg.strictVerify, + ), } } @@ -113,6 +122,35 @@ func (t *treePipeline) mkdirProgrammably(root *Node, cfg *config) error { return t.handlePipelineErr(ctx, errcg, errcm) } +func (t *treePipeline) verify(r io.Reader, cfg *config) error { + ctx, cancel := context.WithCancel(cfg.ctx) + defer cancel() + + t.grower.enableValidation() + splitStream, errcsl := split(ctx, r) + rootStream, errcr := newRootGeneratorPipeline(cfg.space).generate(ctx, splitStream) + growStream, errcg := t.grower.grow(ctx, rootStream) + errcv := t.verifier.verify(ctx, growStream) + return t.handlePipelineErr(ctx, errcsl, errcr, errcg, errcv) +} + +func (t *treePipeline) verifyProgrammably(root *Node, cfg *config) error { + ctx, cancel := context.WithCancel(cfg.ctx) + defer cancel() + + rootStream := make(chan *Node) + go func() { + defer close(rootStream) + rootStream <- root + }() + t.grower.enableValidation() + // when detect invalid node name, return error. process end. + growStream, errcg := t.grower.grow(ctx, rootStream) + // when detected no invalid node name, no output tree. + errcv := t.verifier.verify(ctx, growStream) + return t.handlePipelineErr(ctx, errcg, errcv) +} + // 関心事は各ノードの枝の形成 type growerPipeline interface { grow(context.Context, <-chan *Node) (<-chan *Node, <-chan error) @@ -130,6 +168,12 @@ type mkdirerPipeline interface { mkdir(context.Context, <-chan *Node) <-chan error } +// 関心事はディレクトリの検証 +// interfaceを使う必要はないが、growerPipeline/spreaderPipelineと合わせたいため +type verifierPipeline interface { + verify(context.Context, <-chan *Node) <-chan error +} + // パイプラインの全ステージで最初のエラーを返却 func (*treePipeline) handlePipelineErr(ctx context.Context, echs ...<-chan error) error { eg, ectx := errgroup.WithContext(ctx) diff --git a/pipeline_tree_verifier.go b/pipeline_tree_verifier.go new file mode 100644 index 00000000..90039838 --- /dev/null +++ b/pipeline_tree_verifier.go @@ -0,0 +1,45 @@ +package gtree + +import ( + "context" + "fmt" +) + +type defaultVerifierPipeline struct { + *defaultVerifierSimple +} + +func newVerifierPipeline(dir string, strict bool) verifierPipeline { + return &defaultVerifierPipeline{ + defaultVerifierSimple: newVerifierSimple(dir, strict).(*defaultVerifierSimple), + } +} + +func (dv *defaultVerifierPipeline) verify(ctx context.Context, roots <-chan *Node) <-chan error { + fmt.Println("in verify pipeline") + errc := make(chan error, 1) + + go func() { + defer close(errc) + for { + select { + case <-ctx.Done(): + return + case root, ok := <-roots: + if !ok { + return + } + exists, noExists, err := dv.verifyRoot(root) + if err != nil { + errc <- err + } + if err := dv.handleErr(exists, noExists); err != nil { + errc <- err + } + } + } + }() + + fmt.Println("in verify pipeline end") + return errc +} diff --git a/simple_tree.go b/simple_tree.go index f505219d..eb70a021 100644 --- a/simple_tree.go +++ b/simple_tree.go @@ -12,6 +12,7 @@ type treeSimple struct { grower growerSimple spreader spreaderSimple mkdirer mkdirerSimple + verifier verifierSimple } var _ tree = (*treeSimple)(nil) @@ -35,6 +36,10 @@ func newTreeSimple(cfg *config) tree { return newMkdirerSimple(fileExtensions) } + verifierFactory := func(targetDir string, strict bool) verifierSimple { + return newVerifierSimple(targetDir, strict) + } + return &treeSimple{ grower: growerFactory( cfg.lastNodeFormat, @@ -50,6 +55,10 @@ func newTreeSimple(cfg *config) tree { mkdirer: mkdirerFactory( cfg.fileExtensions, ), + verifier: verifierFactory( + cfg.targetDir, + cfg.strictVerify, + ), } } @@ -100,6 +109,29 @@ func (t *treeSimple) mkdirProgrammably(root *Node, cfg *config) error { return t.mkdirer.mkdir([]*Node{root}) } +func (t *treeSimple) verify(r io.Reader, cfg *config) error { + rg := newRootGeneratorSimple(r, cfg.space) + roots, err := rg.generate() + if err != nil { + return err + } + + t.grower.enableValidation() + if err := t.grower.grow(roots); err != nil { + return err + } + return t.verifier.verify(roots) +} + +func (t *treeSimple) verifyProgrammably(root *Node, cfg *config) error { + t.grower.enableValidation() + // when detect invalid node name, return error. process end. + if err := t.grower.grow([]*Node{root}); err != nil { + return err + } + return t.verifier.verify([]*Node{root}) +} + // 関心事は各ノードの枝の形成 type growerSimple interface { grow([]*Node) error @@ -116,3 +148,9 @@ type spreaderSimple interface { type mkdirerSimple interface { mkdir([]*Node) error } + +// 関心事はディレクトリの検証 +// interfaceを使う必要はないが、growerSimple/spreaderSimpleと合わせたいため +type verifierSimple interface { + verify([]*Node) error +} diff --git a/simple_tree_verifier.go b/simple_tree_verifier.go new file mode 100644 index 00000000..a7a56e0a --- /dev/null +++ b/simple_tree_verifier.go @@ -0,0 +1,127 @@ +package gtree + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" +) + +func newVerifierSimple(dir string, strict bool) verifierSimple { + targetDir := "." + if len(targetDir) != 0 { + targetDir = dir + } + + return &defaultVerifierSimple{ + strict: strict, + targetDir: targetDir, + } +} + +type defaultVerifierSimple struct { + strict bool + targetDir string +} + +// cat testdata/sample9.md | sudo go run cmd/gtree/*.go verify --strict --target-dir /home/ochi/github.com/ddddddO/gtree +// cat testdata/sample10.md | sudo go run cmd/gtree/*.go verify --strict --target-dir /home/ochi/github.com/ddddddO/gtree +func (dv *defaultVerifierSimple) verify(roots []*Node) error { + for i := range roots { + exists, noExists, err := dv.verifyRoot(roots[i]) + if err != nil { + return err + } + if err := dv.handleErr(exists, noExists); err != nil { + return err + } + } + + return nil +} + +func (dv *defaultVerifierSimple) verifyRoot(root *Node) ([]string, []string, error) { + dirsMarkdown := map[string]struct{}{} + if err := dv.recursive(root, dirsMarkdown); err != nil { + return nil, nil, err + } + + dirsFilesystem := map[string]struct{}{} + existDirs := []string{} + rootPath := root.path() + fileSystem := os.DirFS(filepath.Join(dv.targetDir, rootPath)) + err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + dir := filepath.Join(dv.targetDir, rootPath, path) + if _, ok := dirsMarkdown[dir]; !ok { + // Markdownに無いパスがディレクトリに有る => strictモードでエラー + existDirs = append(existDirs, dir) + } + + dirsFilesystem[dir] = struct{}{} + return nil + }) + if err != nil { + return nil, nil, err + } + + // Markdownに有るパスがディレクトリに無い時 => 通常/strictモード共通でエラー + noExistDirs := []string{} + for dir := range dirsMarkdown { + if _, ok := dirsFilesystem[dir]; !ok { + noExistDirs = append(noExistDirs, dir) + } + } + + return existDirs, noExistDirs, nil +} + +func (dv *defaultVerifierSimple) recursive(node *Node, dirs map[string]struct{}) error { + dirs[filepath.Join(dv.targetDir, node.path())] = struct{}{} + + for i := range node.children { + if err := dv.recursive(node.children[i], dirs); err != nil { + return err + } + } + return nil +} + +func (dv *defaultVerifierSimple) handleErr(exists, noExists []string) error { + if (dv.strict && len(exists) != 0) || len(noExists) != 0 { + return VerifyError{ + strict: dv.strict, + exists: exists, + noExists: noExists, + } + } + return nil +} + +type VerifyError struct { + strict bool + exists []string + noExists []string +} + +func (v VerifyError) Error() string { + tabPrefix := func(arr []string) string { + tmp := "" + for i := range arr { + tmp += fmt.Sprintf("\t%s\n", arr[i]) + } + return tmp + } + + msg := "" + if v.strict && len(v.exists) != 0 { + msg += fmt.Sprintf("Extra paths exist:\n%s", tabPrefix(v.exists)) + } + if len(v.noExists) != 0 { + msg += fmt.Sprintf("Required paths does not exist:\n%s", tabPrefix(v.noExists)) + } + return msg +} diff --git a/testdata/sample10.md b/testdata/sample10.md new file mode 100644 index 00000000..82be4b49 --- /dev/null +++ b/testdata/sample10.md @@ -0,0 +1,18 @@ +- example + - find_pipe_programmable-gtree + - main.go + - go-list_pipe_programmable-gtree + - main.go + - like_cli + - adapter + - executor.go + - indentation.go + - main.go + - programmable + - main.go + - noexist + - xxx + +- docs + - main.js + - toast.js diff --git a/testdata/sample9.md b/testdata/sample9.md new file mode 100644 index 00000000..b848e7c2 --- /dev/null +++ b/testdata/sample9.md @@ -0,0 +1,13 @@ +- example + - find_pipe_programmable-gtree + - main.go + - go-list_pipe_programmable-gtree + - main.go + - like_cli + - adapter + - executor.go + - indentation.go + - main.go + - kkk + - programmable + - main.go diff --git a/tree.go b/tree.go index 7b1fa1ec..a8bfe459 100644 --- a/tree.go +++ b/tree.go @@ -9,4 +9,6 @@ type tree interface { outputProgrammably(io.Writer, *Node, *config) error mkdir(io.Reader, *config) error mkdirProgrammably(*Node, *config) error + verify(io.Reader, *config) error + verifyProgrammably(*Node, *config) error } diff --git a/tree_handler.go b/tree_handler.go index f4a9ac37..3d61050e 100644 --- a/tree_handler.go +++ b/tree_handler.go @@ -27,3 +27,14 @@ func Mkdir(r io.Reader, options ...Option) error { } return tree.mkdir(r, cfg) } + +// Verify verifies directories. +func Verify(r io.Reader, options ...Option) error { + cfg := newConfig(options) + + tree := newTreeSimple(cfg) + if cfg.massive { + tree = newTreePipeline(cfg) + } + return tree.verify(r, cfg) +} diff --git a/tree_handler_programmably.go b/tree_handler_programmably.go index c706f10e..0e2ec6cd 100644 --- a/tree_handler_programmably.go +++ b/tree_handler_programmably.go @@ -63,6 +63,26 @@ func MkdirProgrammably(root *Node, options ...Option) error { return tree.mkdirProgrammably(root, cfg) } +// VerifyProgrammably verifies directory. +// This function requires node generated by NewRoot function. +func VerifyProgrammably(root *Node, options ...Option) error { + if root == nil { + return ErrNilNode + } + if !root.isRoot() { + return ErrNotRoot + } + + idxCounter.reset() + cfg := newConfig(options) + + tree := newTreeSimple(cfg) + if cfg.massive { + tree = newTreePipeline(cfg) + } + return tree.verifyProgrammably(root, cfg) +} + // NewRoot creates a starting node for building tree. func NewRoot(text string) *Node { return newNode(text, rootHierarchyNum, idxCounter.next())