Skip to content

Commit

Permalink
upgrade grok
Browse files Browse the repository at this point in the history
  • Loading branch information
@vircoys
vircoys committed Nov 23, 2022
1 parent adc1cc2 commit d65ca91
Show file tree
Hide file tree
Showing 3 changed files with 160 additions and 160 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.18

require (
github.com/DataDog/datadog-agent/pkg/obfuscate v0.39.0
github.com/GuanceCloud/grok v1.1.1
github.com/GuanceCloud/grok v1.1.2
github.com/antchfx/xmlquery v1.3.12
github.com/araddon/dateparse v0.0.0-20201001162425-8aadafed4dc4
github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ github.com/DataDog/datadog-agent/pkg/obfuscate v0.39.0 h1:kXdXGOYeH2pKa3tdNGdrix
github.com/DataDog/datadog-agent/pkg/obfuscate v0.39.0/go.mod h1:MxVcCIC42tBIjPm93BHdh9/vw2LivRiptj3HygI+GGQ=
github.com/DataDog/datadog-go/v5 v5.1.0 h1:Zmq3tCk9+Tdq8Du73M71Zo6Dyx+cEo9QkCSCqQlHFaQ=
github.com/DataDog/datadog-go/v5 v5.1.0/go.mod h1:KhiYb2Badlv9/rofz+OznKoEF5XKTonWyhx5K83AP8E=
github.com/GuanceCloud/grok v1.1.1 h1:dyYyYVn4y0UWFTH3dYV1OZIRDG1wB3Y7aXBZLNqiaiQ=
github.com/GuanceCloud/grok v1.1.1/go.mod h1:AHkJZYf7Qbo1FTZT6htdyScpICpgnkQ5+Hc0EmA88vM=
github.com/GuanceCloud/grok v1.1.2 h1:xscdb39kbPjiO8HfhORt/wDZuQqhi8N/WyjNcQnWujA=
github.com/GuanceCloud/grok v1.1.2/go.mod h1:AHkJZYf7Qbo1FTZT6htdyScpICpgnkQ5+Hc0EmA88vM=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
Expand Down
314 changes: 157 additions & 157 deletions pkg/inimpl/guancecloud/funcs/fn_grok_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,163 +31,163 @@ add_key(grok_match_ok, grok(_, "%{time}")) ; printf("%v", grok_match_ok)`,
expected: true,
outkey: "grok_match_ok",
},
// {
// name: "normal_return_f",
// pl: `
// add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
// add_pattern("_minute", "(?:[0-5][0-9])")
// add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
// add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
// add_key(grok_match_ok, grok(_, "%{time}"))`,
// in: "12 :13:14.123",
// expected: false,
// outkey: "grok_match_ok",
// },
// {
// name: "normal_return_sample_t",
// pl: `
// add_key(grok_match_ok, grok(_, "12 :13:14.123"))`,
// in: "12 :13:14.123",
// expected: true,
// outkey: "grok_match_ok",
// },
// {
// name: "normal_return_sample_f",
// pl: `
// add_key(grok_match_ok, grok(_, "12 :13:14.123"))`,
// in: "12:13:14.123",
// expected: false,
// outkey: "grok_match_ok",
// },
// {
// name: "normal",
// pl: `
// add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
// add_pattern("_minute", "(?:[0-5][0-9])")
// add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
// add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
// grok(_, "%{time}")`,
// in: "12:13:14.123",
// expected: "14.123",
// outkey: "second",
// },
// {
// name: "normal",
// pl: `
// add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
// add_pattern("_minute", "(?:[0-5][0-9])")
// add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
// add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
// grok(_, "%{time}")`,
// in: "12:13:14",
// expected: "13",
// outkey: "minute",
// },
// {
// name: "normal",
// pl: `
// add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
// add_pattern("_minute", "(?:[0-5][0-9])")
// add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
// add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
// grok(_, "%{time}")`,
// in: "12:13:14",
// expected: "12",
// outkey: "hour",
// },
// {
// name: "normal",
// pl: `
// add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
// add_pattern("_minute", "(?:[0-5][0-9])")
// add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
// add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
// grok(_, "%{time}")`,
// in: "12:13:14",
// expected: "14",
// outkey: "second",
// },
// {
// name: "normal",
// pl: `
// add_pattern("time", "%{NUMBER:time:float}")
// grok(_, '''%{time}
// %{WORD:word:string}
// %{WORD:code:int}
// %{WORD:w1}''')`,
// in: `1.1
// s
// 123cvf
// aa222`,
// expected: int64(0),
// outkey: "code",
// },
// {
// name: "normal",
// pl: `
// add_pattern("time", "%{NUMBER:time:float}")
// grok(_, '''%{time}
// %{WORD:word:string}
// %{WORD:code:int}
// %{WORD:w1}''')`,
// in: `1.1
// s
// 123
// aa222`,
// expected: int64(123),
// outkey: "code",
// },
// {
// name: "normal",
// pl: `
// add_pattern("time", "%{NUMBER:time:float}")
// grok(_, '''%{time}
// %{WORD:word:str}
// %{WORD:code:int}
// %{WORD:w1}''')`,
// in: `1.1
// s
// 123
// aa222`,
// expected: int64(123),
// outkey: "code",
// },
// {
// name: "normal",
// pl: `
// add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
// add_pattern("_minute", "(?:[0-5][0-9])")
// add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
// add_pattern("time", "([^0-9]?)%{_hour:hour:string}:%{_minute:minute:int}(?::%{_second:second:float})([^0-9]?)")
// grok(_, "%{WORD:date} %{time}")`,
// in: "2021/1/11 2:13:14.123",
// expected: float64(14.123),
// outkey: "second",
// },
// {
// name: "trim_space",
// in: " not_space ",
// pl: `add_pattern("d", "[\\s\\S]*")
// grok(_, "%{d:item}")`,
// expected: "not_space",
// outkey: "item",
// },
// {
// name: "trim_space, enable",
// in: " not_space ",
// pl: `add_pattern("d", "[\\s\\S]*")
// grok(_, "%{d:item}", true)`,
// expected: "not_space",
// outkey: "item",
// },
// {
// name: "trim_space, disable",
// in: " not_space ",
// pl: `add_pattern("d", "[\\s\\S]*")
// grok(_, "%{d:item}", false)`,
// expected: " not_space ",
// outkey: "item",
// },
{
name: "normal_return_f",
pl: `
add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
add_pattern("_minute", "(?:[0-5][0-9])")
add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
add_key(grok_match_ok, grok(_, "%{time}"))`,
in: "12 :13:14.123",
expected: false,
outkey: "grok_match_ok",
},
{
name: "normal_return_sample_t",
pl: `
add_key(grok_match_ok, grok(_, "12 :13:14.123"))`,
in: "12 :13:14.123",
expected: true,
outkey: "grok_match_ok",
},
{
name: "normal_return_sample_f",
pl: `
add_key(grok_match_ok, grok(_, "12 :13:14.123"))`,
in: "12:13:14.123",
expected: false,
outkey: "grok_match_ok",
},
{
name: "normal",
pl: `
add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
add_pattern("_minute", "(?:[0-5][0-9])")
add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
grok(_, "%{time}")`,
in: "12:13:14.123",
expected: "14.123",
outkey: "second",
},
{
name: "normal",
pl: `
add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
add_pattern("_minute", "(?:[0-5][0-9])")
add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
grok(_, "%{time}")`,
in: "12:13:14",
expected: "13",
outkey: "minute",
},
{
name: "normal",
pl: `
add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
add_pattern("_minute", "(?:[0-5][0-9])")
add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
grok(_, "%{time}")`,
in: "12:13:14",
expected: "12",
outkey: "hour",
},
{
name: "normal",
pl: `
add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
add_pattern("_minute", "(?:[0-5][0-9])")
add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)")
grok(_, "%{time}")`,
in: "12:13:14",
expected: "14",
outkey: "second",
},
{
name: "normal",
pl: `
add_pattern("time", "%{NUMBER:time:float}")
grok(_, '''%{time}
%{WORD:word:string}
%{WORD:code:int}
%{WORD:w1}''')`,
in: `1.1
s
123cvf
aa222`,
expected: int64(0),
outkey: "code",
},
{
name: "normal",
pl: `
add_pattern("time", "%{NUMBER:time:float}")
grok(_, '''%{time}
%{WORD:word:string}
%{WORD:code:int}
%{WORD:w1}''')`,
in: `1.1
s
123
aa222`,
expected: int64(123),
outkey: "code",
},
{
name: "normal",
pl: `
add_pattern("time", "%{NUMBER:time:float}")
grok(_, '''%{time}
%{WORD:word:str}
%{WORD:code:int}
%{WORD:w1}''')`,
in: `1.1
s
123
aa222`,
expected: int64(123),
outkey: "code",
},
{
name: "normal",
pl: `
add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)")
add_pattern("_minute", "(?:[0-5][0-9])")
add_pattern("_hour", "(?:2[0123]|[01]?[0-9])")
add_pattern("time", "([^0-9]?)%{_hour:hour:string}:%{_minute:minute:int}(?::%{_second:second:float})([^0-9]?)")
grok(_, "%{WORD:date} %{time}")`,
in: "2021/1/11 2:13:14.123",
expected: float64(14.123),
outkey: "second",
},
{
name: "trim_space",
in: " not_space ",
pl: `add_pattern("d", "[\\s\\S]*")
grok(_, "%{d:item}")`,
expected: "not_space",
outkey: "item",
},
{
name: "trim_space, enable",
in: " not_space ",
pl: `add_pattern("d", "[\\s\\S]*")
grok(_, "%{d:item}", true)`,
expected: "not_space",
outkey: "item",
},
{
name: "trim_space, disable",
in: " not_space ",
pl: `add_pattern("d", "[\\s\\S]*")
grok(_, "%{d:item}", false)`,
expected: " not_space ",
outkey: "item",
},
}

for idx, tc := range cases {
Expand Down

0 comments on commit d65ca91

Please sign in to comment.