Skip to content

Commit

Permalink
大量root用処理をスイッチできるように (#179)
Browse files Browse the repository at this point in the history
  • Loading branch information
ddddddO authored Jun 24, 2023
1 parent 3b53bec commit f1f4743
Show file tree
Hide file tree
Showing 21 changed files with 1,064 additions and 297 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ root4/
root5/
root6/
root7/
root8/
root_a/
root_b/
root_c/
Expand All @@ -20,4 +21,5 @@ root_f/
root_g/
root_h/
root_i/
root_j/
Primate/
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
sweep:
rm -rf ./root/ ./root1/ ./root2/ ./root3/ ./root4/ ./root5/ ./root6/ ./root7/ Primate/ gtree/
rm -rf ./root_a/ ./root_b/ ./root_c/ ./root_d/ ./root_e/ ./root_f/ ./root_g/ ./root_h/ ./root_i/
rm -rf ./root/ ./root1/ ./root2/ ./root3/ ./root4/ ./root5/ ./root6/ ./root7/ ./root8/ Primate/ gtree/
rm -rf ./root_a/ ./root_b/ ./root_c/ ./root_d/ ./root_e/ ./root_f/ ./root_g/ ./root_h/ ./root_i/ ./root_j/

fmt: sweep
go fmt ./...
Expand Down
20 changes: 13 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ OPTIONS:
--file value, -f value specify the path to markdown file. (default: stdin)
--two-spaces, --ts set this option when the markdown indent is 2 spaces. (default: tab spaces)
--four-spaces, --fs set this option when the markdown indent is 4 spaces. (default: tab spaces)
--massive, -m set this option when there are very many blocks of markdown. (default: false)
--json, -j set this option when outputting JSON. (default: tree)
--yaml, -y set this option when outputting YAML. (default: tree)
--toml, -t set this option when outputting TOML. (default: tree)
Expand Down Expand Up @@ -400,6 +401,7 @@ OPTIONS:
--file value, -f value specify the path to markdown file. (default: stdin)
--two-spaces, --ts set this option when the markdown indent is 2 spaces. (default: tab spaces)
--four-spaces, --fs set this option when the markdown indent is 4 spaces. (default: tab spaces)
--massive, -m set this option when there are very many blocks of markdown. (default: false)
--dry-run, -d, --dr dry run. detects node that is invalid for directory generation.
the order of the output and made directories does not always match. (default: false)
--extension value, -e value, --ext value [ --extension value, -e value, --ext value ] set this option if you want to create file instead of directory.
Expand Down Expand Up @@ -951,27 +953,31 @@ func main() {

# Process

> **Note**<br>
> This process is for the Massive Roots mode.
## e.g. [*gtree/tree_handler.go*](https://github.com/ddddddO/gtree/blob/master/tree_handler.go)
## e.g. [*gtree/pipeline_tree.go*](https://github.com/ddddddO/gtree/blob/master/pipeline_tree.go)

<image src="./process.svg" width=100%>


# Performance


> **Warning**<br>
> Depends on the environment.
- Comparison before and after software architecture was changed.
- In the case of few Roots, previous architecture is faster in execution😅
- However, for multiple Roots, execution speed tends to be faster💪✨
- Comparison simple implementation and pipeline implementation.
- In the case of few Roots, simple implementation is faster in execution!
- Use this one by default.
- However, for multiple Roots, pipeline implementation execution speed tends to be faster💪✨
- In the CLI, it is available by specifying `--massive`.
- In the Go program, it is available by specifying `WithMassive` func.

<image src="./performance.svg" width=100%>

<details><summary>Benchmark log</summary>

## Before pipelining
## Simple implementation
```console
$ go test -benchmem -bench Benchmark -benchtime 100x tree_handler_benchmark_test.go
goos: linux
Expand All @@ -991,7 +997,7 @@ PASS
ok command-line-arguments 68.124s
```

## After pipelining
## Pipeline implementation
```console
$ go test -benchmem -bench Benchmark -benchtime 100x tree_handler_benchmark_test.go
goos: linux
Expand Down
5 changes: 5 additions & 0 deletions cmd/gtree/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ func main() {
Usage: "set this option when the markdown indent is 4 spaces.",
DefaultText: "tab spaces",
},
&cli.BoolFlag{
Name: "massive",
Aliases: []string{"m"},
Usage: "set this option when there are very many blocks of markdown.",
},
}

outputFlags := []cli.Flag{
Expand Down
14 changes: 12 additions & 2 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ type config struct {
intermedialNodeFormat branchFormat

space spaceType
massive bool
encode encode
dryrun bool
fileExtensions []string
Expand All @@ -20,8 +21,9 @@ func newConfig(options []Option) (*config, error) {
directly: "├──",
indirectly: "│ ",
},
space: spacesTab,
encode: encodeDefault,
space: spacesTab,
massive: false,
encode: encodeDefault,
}
for _, opt := range options {
if opt == nil {
Expand Down Expand Up @@ -72,6 +74,14 @@ func WithBranchFormatLastNode(directly, indirectly string) Option {
}
}

// WithMassive returns function for large amount roots.
func WithMassive() Option {
return func(c *config) error {
c.massive = true
return nil
}
}

// WithEncodeJSON returns function for output json format.
func WithEncodeJSON() Option {
return func(c *config) error {
Expand Down
158 changes: 158 additions & 0 deletions pipeline_tree.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
//go:build !wasm

package gtree

import (
"context"
"io"

"github.com/fatih/color"
"golang.org/x/sync/errgroup"
)

type treePipeline struct {
grower growerPipeline
spreader spreaderPipeline
mkdirer mkdirerPipeline
}

func newTreePipeline(conf *config) *treePipeline {
growerFactory := func(lastNodeFormat, intermedialNodeFormat branchFormat, dryrun bool, encode encode) growerPipeline {
if encode != encodeDefault {
return newNopGrowerPipeline()
}
return newGrowerPipeline(lastNodeFormat, intermedialNodeFormat, dryrun)
}

spreaderFactory := func(encode encode, dryrun bool, fileExtensions []string) spreaderPipeline {
if dryrun {
return newColorizeSpreaderPipeline(fileExtensions)
}
return newSpreaderPipeline(encode)
}

mkdirerFactory := func(fileExtensions []string) mkdirerPipeline {
return newMkdirerPipeline(fileExtensions)
}

return &treePipeline{
grower: growerFactory(
conf.lastNodeFormat,
conf.intermedialNodeFormat,
conf.dryrun,
conf.encode,
),
spreader: spreaderFactory(
conf.encode,
conf.dryrun,
conf.fileExtensions,
),
mkdirer: mkdirerFactory(
conf.fileExtensions,
),
}
}

func (t *treePipeline) output(w io.Writer, r io.Reader, conf *config) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

splitStream, errcsl := split(ctx, r)
rootStream, errcr := newRootGeneratorPipeline(conf.space).generate(ctx, splitStream)
growStream, errcg := t.grow(ctx, rootStream)
errcs := t.spread(ctx, w, growStream)
return t.handlePipelineErr(errcsl, errcr, errcg, errcs)
}

func (t *treePipeline) outputProgrammably(w io.Writer, root *Node, conf *config) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

rootStream := make(chan *Node)
go func() {
defer close(rootStream)
rootStream <- root
}()
growStream, errcg := t.grow(ctx, rootStream)
errcs := t.spread(ctx, w, growStream)
return t.handlePipelineErr(errcg, errcs)
}

func (t *treePipeline) makedir(r io.Reader, conf *config) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

splitStream, errcsl := split(ctx, r)
rootStream, errcr := newRootGeneratorPipeline(conf.space).generate(ctx, splitStream)
growStream, errcg := t.grow(ctx, rootStream)
errcm := t.mkdir(ctx, growStream)
return t.handlePipelineErr(errcsl, errcr, errcg, errcm)
}

func (t *treePipeline) makedirProgrammably(root *Node, conf *config) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

rootStream := make(chan *Node)
go func() {
defer close(rootStream)
rootStream <- root
}()
t.enableValidation()
// when detect invalid node name, return error. process end.
growStream, errcg := t.grow(ctx, rootStream)
if conf.dryrun {
// when detected no invalid node name, output tree.
errcs := t.spread(ctx, color.Output, growStream)
return t.handlePipelineErr(errcg, errcs)
}
// when detected no invalid node name, no output tree.
errcm := t.mkdir(ctx, growStream)
return t.handlePipelineErr(errcg, errcm)
}

// 関心事は各ノードの枝の形成
type growerPipeline interface {
grow(context.Context, <-chan *Node) (<-chan *Node, <-chan error)
enableValidation()
}

// 関心事はtreeの出力
type spreaderPipeline interface {
spread(context.Context, io.Writer, <-chan *Node) <-chan error
}

// 関心事はファイルの生成
// interfaceを使う必要はないが、growerPipeline/spreaderPipelineと合わせたいため
type mkdirerPipeline interface {
mkdir(context.Context, <-chan *Node) <-chan error
}

func (t *treePipeline) grow(ctx context.Context, roots <-chan *Node) (<-chan *Node, <-chan error) {
return t.grower.grow(ctx, roots)
}

func (t *treePipeline) spread(ctx context.Context, w io.Writer, roots <-chan *Node) <-chan error {
return t.spreader.spread(ctx, w, roots)
}

func (t *treePipeline) mkdir(ctx context.Context, roots <-chan *Node) <-chan error {
return t.mkdirer.mkdir(ctx, roots)
}

// パイプラインの全ステージで最初のエラーを返却
func (*treePipeline) handlePipelineErr(echs ...<-chan error) error {
eg, _ := errgroup.WithContext(context.TODO())
for i := range echs {
i := i
eg.Go(func() error {
for e := range echs[i] {
if e != nil {
return e
}
}
return nil
})
}
return eg.Wait()
}
Loading

0 comments on commit f1f4743

Please sign in to comment.