Merge remote-tracking branch 'origin/topic/awelzel/conn-cut'

.github/workflows/check.yml

@@ -10,14 +10,14 @@ jobs:
     strategy:
       matrix:
         version:
-          - zeek:5.0.4
-          - zeek:5.1.1
+          - zeek:5.0
+          - zeek:6.0
           - zeek-dev:latest
 
       fail-fast: false
 
     runs-on: ubuntu-latest
-    container: zeekurity/${{ matrix.version }}
+    container: zeek/${{ matrix.version }}
 
     steps:
       - uses: actions/checkout@v2

tests/analyzer/basic.zeek

@@ -2,10 +2,12 @@
 
 # @TEST-EXEC: zeek -r ${TRACES}/http-post.pcap frameworks/files/hash-all-files %INPUT
 # @TEST-EXEC: cat files.log | sed 's/SHA1,MD5/MD5,SHA1/g' >files.log.tmp && mv -f files.log.tmp files.log
+# @TEST-EXEC: zeek-cut -C ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service history <conn.log >conn.log2 && mv conn.log2 conn.log
 # @TEST-EXEC: zeek-cut -C fuid source depth analyzers mime_type filename total_bytes <files.log >files.log.tmp && mv files.log.tmp files.log
 # @TEST-EXEC: btest-diff conn.log
-# @TEST-EXEC: btest-diff http.log
-# @TEST-EXEC: btest-diff files.log
+# Skip baselining of fuids on pre-6.0 versions (fuids stopped being canonified with 6.0).
+# @TEST-EXEC: zeek -b -e 'exit(Version::at_least("6.0") ? 1 : 0)' || btest-diff http.log
+# @TEST-EXEC: zeek -b -e 'exit(Version::at_least("6.0") ? 1 : 0)' || btest-diff files.log
 #
 # @TEST-DOC: Test HTTP analyzer with small trace.
 

tests/analyzer/dpd.zeek

@@ -1,8 +1,9 @@
 # Copyright (c) 2021 by the Zeek Project. See LICENSE for details.
 
 # @TEST-EXEC: zeek -Cr ${TRACES}/http-non-default-port.pcap %INPUT
+# @TEST-EXEC: zeek-cut -C ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service history <conn.log >conn.log2 && mv conn.log2 conn.log
 # @TEST-EXEC: btest-diff conn.log
-# @TEST-EXEC: mv http.log http.log.bak && zeek-cut -n host <http.log.bak >http.log && btest-diff http.log
+# @TEST-EXEC: mv http.log http.log.bak && zeek-cut -n host orig_fuids resp_fuids <http.log.bak >http.log && btest-diff http.log
 #
 # @TEST-DOC: Test that DPD triggers the HTTP analyzer with communication on non-default port.
 

tests/baseline/analyzer.basic/conn.log

@@ -6,7 +6,7 @@
 #unset_field	-
 #path	conn
 #open XXXX-XX-XX-XX-XX-XX
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	history
+#types	time	string	addr	port	addr	port	enum	string	string
 #close XXXX-XX-XX-XX-XX-XX
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	53595	54.243.55.129	80	tcp	http	0.068875	160	519	SF	-	-	0	ShADadFf	8	588	6	839	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	53595	54.243.55.129	80	tcp	http	ShADadFf

tests/baseline/analyzer.basic/files.log

@@ -9,5 +9,5 @@
 #fields	fuid	source	depth	analyzers	mime_type	filename	total_bytes
 #types	string	string	count	set[string]	string	string	count
 #close XXXX-XX-XX-XX-XX-XX
-XXXXXXXXXXXXXXXXX	HTTP	0	MD5,SHA1	text/json	-	366
-XXXXXXXXXXXXXXXXX	HTTP	0	MD5,SHA1	text/plain	-	11
+FM47gX3vI5ofQPm1li	HTTP	0	MD5,SHA1	text/plain	-	11
+FZjUS57tUkGFTibv3	HTTP	0	MD5,SHA1	text/json	-	366

tests/baseline/analyzer.basic/http.log

@@ -9,4 +9,4 @@
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]
 #close XXXX-XX-XX-XX-XX-XX
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	53595	54.243.55.129	80	1	POST	httpbin.org	/post	-	1.1	curl/7.29.0	-	11	366	200	OK	-	-	(empty)	-	-	-	XXXXXXXXXXXXXXXXX	-	text/plain	XXXXXXXXXXXXXXXXX	-	text/json
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	141.142.228.5	53595	54.243.55.129	80	1	POST	httpbin.org	/post	-	1.1	curl/7.29.0	-	11	366	200	OK	-	-	(empty)	-	-	-	FM47gX3vI5ofQPm1li	-	text/plain	FZjUS57tUkGFTibv3	-	text/json

tests/baseline/analyzer.dpd/conn.log

@@ -6,7 +6,7 @@
 #unset_field	-
 #path	conn
 #open XXXX-XX-XX-XX-XX-XX
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	history
+#types	time	string	addr	port	addr	port	enum	string	string
 #close XXXX-XX-XX-XX-XX-XX
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51424	127.0.0.1	1234	tcp	http	0.002493	135	451	SF	-	-	0	ShAaDdfF	6	459	6	775	-
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51424	127.0.0.1	1234	tcp	http	ShAaDdfF

tests/baseline/analyzer.dpd/http.log

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 ### NOTE: This file has been sorted with diff-sort.
-XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51424	127.0.0.1	1234	1	GET	/	-	1.0	HTTPie/2.5.0	-	0	297	200	OK	-	-	(empty)	-	-	-	-	-	-	XXXXXXXXXXXXXXXXX	-	text/html
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	51424	127.0.0.1	1234	1	GET	/	-	1.0	HTTPie/2.5.0	-	0	297	200	OK	-	-	(empty)	-	-	-	-	-	-	text/html