-
Notifications
You must be signed in to change notification settings - Fork 39
/
build.js
27 lines (22 loc) · 698 Bytes
/
build.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
var fs = require('fs')
var path = require('path')
var https = require('https')
var concat = require('concat-stream')
var bail = require('bail')
var root = 'corpus'
var endpoint =
'https://raw.githubusercontent.com/lorenbrichter/Words/master/Words/en.txt'
https.request(endpoint, onrequest).end()
function onrequest(res) {
res.pipe(concat(onconcat)).on('error', bail)
}
function onconcat(buf) {
var data = fs
.readdirSync(root)
.flatMap(d => String(fs.readFileSync(path.join(root, d))).split('\n'))
.concat(String(buf).split('\n'))
.map(d => d.toLowerCase())
.filter(Boolean)
.sort()
fs.writeFile('index.json', JSON.stringify([...new Set(data)]) + '\n', bail)
}