Skip to content

Commit

Permalink
Merge branch 'bro-logs'
Browse files Browse the repository at this point in the history 
- Added support for Bro/Zeek HTTP logs (fixes #71)
- Fixes to Bro/Zeek DNS and IP log reader
  • Loading branch information
@tg
tg committed Oct 7, 2019
2 parents 14e46d4 + 18e928b commit dc371f1
Show file tree
Hide file tree
Showing 379 changed files with 161,324 additions and 41,119 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- run:
name: install golang
command: |
wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz -O /tmp/golang.tar.gz
wget https://dl.google.com/go/go1.11.11.linux-amd64.tar.gz -O /tmp/golang.tar.gz
tar -C /usr/local -xzf /tmp/golang.tar.gz
ln -s /usr/local/go/bin/go /usr/local/bin/
- checkout
Expand Down
5 changes: 4 additions & 1 deletion config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,10 @@ func (cfg *Config) validate() error {
invalidTypeFormat = true
}
case "http":
if monitor.Format != "suricata" {
switch monitor.Format {
case "suricate", "bro":
// ok
default:
invalidTypeFormat = true
}
default:
Expand Down
178 changes: 143 additions & 35 deletions logs/bro/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ type Parser struct {
separator string
setSeparator string
emptyField string
unsetFiled string
fileds []string
unsetField string
fields []string
}
}

Expand Down Expand Up @@ -52,6 +52,13 @@ func NewFileParser(filename string) (*Parser, error) {
return p, nil
}

func (p *Parser) nonEmpty(f string) string {
if f == p.metadata.emptyField || f == p.metadata.unsetField {
return ""
}
return f
}

// ReadDNS reads all dns packets from the file.
func (p *Parser) ReadDNS() ([]*packet.DNSPacket, error) {
if p.r == nil {
Expand Down Expand Up @@ -120,36 +127,42 @@ func (p *Parser) ParseLineDNS(line string) (*packet.DNSPacket, error) {
}

// get values for one entry
fileds := strings.Split(line, p.metadata.separator)
if len(fileds) != len(p.metadata.fileds) {
fields := strings.Split(line, p.metadata.separator)
if len(fields) != len(p.metadata.fields) {
return nil, fmt.Errorf("bro dns log invalid entry at line: %q", line)
}

var dnspacket packet.DNSPacket

// parse values based on fileds
for i, f := range p.metadata.fileds {
// parse values based on fields
for i, f := range p.metadata.fields {
switch f {
case "ts":
timestamp, err := parseEpochTime(fileds[i])
timestamp, err := parseEpochTime(fields[i])
if err != nil {
return nil, fmt.Errorf("bro dns log invalid timestamp: %s", err)
}
dnspacket.Timestamp = timestamp
case "id.orig_h":
dnspacket.SrcIP = net.ParseIP(fileds[i])
dnspacket.SrcIP = net.ParseIP(fields[i])
case "id.orig_p":
port, err := strconv.ParseUint(fields[i], 10, 16)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid port at line: %q: %s", line, err)
}
dnspacket.SrcPort = int(port)
case "query":
dnspacket.FQDN = fileds[i]
dnspacket.FQDN = fields[i]
case "qtype_name":
dnspacket.RecordType = fileds[i]
dnspacket.RecordType = p.nonEmpty(fields[i])
case "id.resp_p":
port, err := strconv.ParseInt(fileds[i], 10, 8)
port, err := strconv.ParseUint(fields[i], 10, 16)
if err != nil {
return nil, fmt.Errorf("bro dns log invalid port at line: %q", line)
}
dnspacket.DstPort = int(port)
case "proto":
dnspacket.Protocol = strings.ToLower(fileds[i])
dnspacket.Protocol = strings.ToLower(fields[i])
}
}

Expand All @@ -170,47 +183,47 @@ func (p *Parser) ParseLineIP(line string) (*packet.IPPacket, error) {
}

// get values for one entry
fileds := strings.Split(line, p.metadata.separator)
if len(fileds) != len(p.metadata.fileds) {
fields := strings.Split(line, p.metadata.separator)
if len(fields) != len(p.metadata.fields) {
return nil, fmt.Errorf("conn bro log - invalid entry at line: %q", line)
}

var ippacket packet.IPPacket

// parse values based on fileds
for i, f := range p.metadata.fileds {
// parse values based on fields
for i, f := range p.metadata.fields {
switch f {
case "ts":
timestamp, err := parseEpochTime(fileds[i])
timestamp, err := parseEpochTime(fields[i])
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid timestamp: %s", err)
}
ippacket.Timestamp = timestamp
case "id.orig_h":
ippacket.SrcIP = net.ParseIP(fileds[i])
ippacket.SrcIP = net.ParseIP(fields[i])
case "id.orig_p":
port, err := strconv.ParseInt(fileds[i], 10, 8)
port, err := strconv.ParseUint(fields[i], 10, 16)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid port at line: %q", line)
}
ippacket.SrcPort = int(port)
case "id.resp_h":
ippacket.DstIP = net.ParseIP(fileds[i])
ippacket.DstIP = net.ParseIP(fields[i])
case "id.resp_p":
port, err := strconv.ParseInt(fileds[i], 10, 8)
port, err := strconv.ParseUint(fields[i], 10, 16)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid port at line: %q", line)
}
ippacket.DstPort = int(port)
case "proto":
ippacket.Protocol = strings.ToLower(fileds[i])
ippacket.Protocol = strings.ToLower(fields[i])
case "orig_bytes":
fallthrough
case "orig_ip_bytes":
if fileds[i] != p.metadata.unsetFiled {
count, err := strconv.ParseInt(fileds[i], 10, 8)
if fields[i] != p.metadata.unsetField {
count, err := strconv.ParseInt(fields[i], 10, 64)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid orig bytes count at line: %q %q %q", line, fileds[i], p.metadata.unsetFiled)
return nil, fmt.Errorf("conn bro log - invalid orig bytes count at line: %q %q %q", line, fields[i], p.metadata.unsetField)
}
if count > 0 {
ippacket.BytesCount += int(count)
Expand All @@ -220,8 +233,8 @@ func (p *Parser) ParseLineIP(line string) (*packet.IPPacket, error) {
case "resp_bytes":
fallthrough
case "resp_ip_bytes":
if fileds[i] != p.metadata.unsetFiled {
count, err := strconv.ParseInt(fileds[i], 10, 8)
if fields[i] != p.metadata.unsetField {
count, err := strconv.ParseInt(fields[i], 10, 64)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid resp bytes count at line: %q", line)
}
Expand All @@ -231,19 +244,114 @@ func (p *Parser) ParseLineIP(line string) (*packet.IPPacket, error) {
}
}
case "ja3":
ippacket.Ja3 = fileds[i]
ippacket.Ja3 = p.nonEmpty(fields[i])
}
}

return &ippacket, nil
}

func (*Parser) ReadHTTP() ([]*client.HTTPEntry, error) {
return nil, nil
func (p *Parser) ReadHTTP() ([]*client.HTTPEntry, error) {
if p.r == nil {
return nil, fmt.Errorf("bro parser must be created with file reader")
}

var packets []*client.HTTPEntry

s := bufio.NewScanner(p.r)
for s.Scan() {
dnspacket, err := p.ParseLineHTTP(string(s.Bytes()))
if err != nil {
return nil, err
}
if dnspacket != nil {
packets = append(packets, dnspacket)
}
}

if err := s.Err(); err != nil {
return nil, err
}
return packets, nil

}

func (*Parser) ParseLineHTTP(line string) (*client.HTTPEntry, error) {
return nil, nil
func (p *Parser) ParseLineHTTP(line string) (*client.HTTPEntry, error) {
line = strings.TrimSpace(line)
if len(line) == 0 {
return nil, nil
}
if line[0] == '#' {
if err := p.readMetadata(line); err != nil {
return nil, err
}
return nil, nil
}

// get values for one entry
fields := strings.Split(line, p.metadata.separator)
if len(fields) != len(p.metadata.fields) {
return nil, fmt.Errorf("bro dns log invalid entry at line: %q", line)
}

var (
entry client.HTTPEntry
host, uri string
)

// parse values based on fields
for i, f := range p.metadata.fields {
switch f {
case "ts":
timestamp, err := parseEpochTime(fields[i])
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid timestamp: %s", err)
}
entry.Timestamp = timestamp
case "id.orig_h":
entry.SrcIP = net.ParseIP(fields[i])
case "id.orig_p":
port, err := strconv.ParseUint(fields[i], 10, 16)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid port at line: %q", line)
}
entry.SrcPort = uint16(port)
case "method":
entry.Method = fields[i]
case "host":
host = fields[i]
case "uri":
uri = fields[i]
case "referrer":
entry.Referrer = p.nonEmpty(fields[i])
case "user_agent":
entry.UserAgent = p.nonEmpty(fields[i])
case "request_body_len":
count, err := strconv.ParseInt(fields[i], 10, 63)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid request_body_len at line: %q %q %q", line, fields[i], p.metadata.unsetField)
}
entry.BytesOut = count
case "response_body_len":
count, err := strconv.ParseInt(fields[i], 10, 63)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid response_body_len at line: %q %q %q", line, fields[i], p.metadata.unsetField)
}
entry.BytesIn = count
case "status_code":
code, err := strconv.ParseUint(fields[i], 10, 16)
if err != nil {
return nil, fmt.Errorf("conn bro log - invalid status_code at line: %q %q %q", line, fields[i], p.metadata.unsetField)
}
entry.Status = int(code)
case "resp_mime_types":
entry.ContentType = p.nonEmpty(fields[i])
}
}

entry.URL = fmt.Sprintf("http://%s%s", host, uri)

return &entry, nil
}

// Close underlying log file.
Expand All @@ -253,7 +361,7 @@ func (p *Parser) Close() error {

// reads metadata from bro file.
func (p *Parser) readMetadata(line string) error {
// sometimes metadata needs to be reload
// sometimes metadata needs to be reloaded
// if line starts with set_separator then clear metadata separator
if strings.HasPrefix(line, "#separator") {
p.metadata.separator = " "
Expand All @@ -279,9 +387,9 @@ func (p *Parser) readMetadata(line string) error {
case "empty_field":
p.metadata.emptyField = metadata[1]
case "unset_field":
p.metadata.unsetFiled = metadata[1]
p.metadata.unsetField = metadata[1]
case "fields":
p.metadata.fileds = strings.Split(metadata[1], p.metadata.separator)
p.metadata.fields = strings.Split(metadata[1], p.metadata.separator)
}

return nil
Expand Down
Loading

0 comments on commit dc371f1

Please sign in to comment.