forked from hjhee/tiebaSpider
-
Notifications
You must be signed in to change notification settings - Fork 0
/
template.go
118 lines (102 loc) · 2.65 KB
/
template.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
package main
import (
"bufio"
"encoding/json"
"fmt"
"html/template"
"os"
"regexp"
"sort"
"sync"
)
func writeOutput(filename string, callback func(w *bufio.Writer) error) error {
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("error creating output file %s: %v", filename, err)
}
defer f.Close()
w := bufio.NewWriter(f)
err = callback(w)
if err != nil {
return fmt.Errorf("error writing to bufio %s, %v", filename, err)
}
err = w.Flush()
if err != nil {
return err
}
return nil
}
func renderHTML(done <-chan struct{}, tempc <-chan *TemplateField, tmpl *template.Template) (chan string, chan error) {
outputc := make(chan string)
errc := make(chan error)
// spawn renderers
var wg sync.WaitGroup
wg.Add(config.NumRenderer)
for i := 0; i < config.NumRenderer; i++ {
go func() {
defer wg.Done()
for {
select {
case <-done:
return
case t, ok := <-tempc:
if !ok {
return // no new task from parser, exit
}
sort.Slice(t.Comments, func(a, b int) bool {
return t.Comments[a].PostNO < t.Comments[b].PostNO
})
for _, v := range t.Lzls.Map {
sort.Slice(v.Info, func(a, b int) bool {
return v.Info[a].Index < v.Info[b].Index
})
}
// no longer merge as requested in issue #8
// t.Merge()
// t.Unique()
// #6: remove illegal character in title
// ref: https://www.codeproject.com/tips/758861/removing-characters-which-are-not-allowed-in-windo
filenameRegex := regexp.MustCompile(`[\\/:*?""<>|]`)
validFilename := filenameRegex.ReplaceAllLiteralString(t.Title, "")
filename := fmt.Sprintf("output/file_%s.json", validFilename)
b, err := json.Marshal(t)
if err != nil {
errc <- err
continue
}
err = writeOutput(filename, func(w *bufio.Writer) error {
_, err := w.Write(b)
return err
})
if err != nil {
errc <- err
continue
}
filename = fmt.Sprintf("output/file_%s.html", validFilename)
err = writeOutput(filename, func(w *bufio.Writer) error {
if err := tmpl.Execute(w, struct {
Title string
Url string
Comments []*OutputField
Lzls map[uint64]*LzlComment
}{Title: t.Title, Url: t.Url, Comments: t.Comments, Lzls: t.Lzls.Map}); err != nil {
return fmt.Errorf("error executing template %s: %v", filename, err)
}
return nil
})
if err != nil {
errc <- err
continue
}
outputc <- filename // report finished task
}
}
}()
}
go func() {
wg.Wait()
close(errc)
close(outputc)
}()
return outputc, errc
}