From 24d538ed9053b0df66e7766d7033ef5bad8cbb90 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Mon, 4 Nov 2024 16:55:10 -0800 Subject: [PATCH 1/3] Update file experiments to the use of logrotate FS This commit updates the file_to_blackhole experiments to rely on the lading logrotate FS generator, allowing us to assert on whether the log Agent loses bytes in the course of its operation. I've added a new check asserting that no bytes are lost, although I do not expect this to pass on higher latency intake experiments yet. REF https://github.com/DataDog/datadog-agent/pull/30744 REF https://github.com/DataDog/lading/pull/1090 Signed-off-by: Brian L. Troutwine --- .../datadog-agent/conf.d/disk-listener.d/conf.yaml | 2 +- .../file_to_blackhole_0ms_latency/experiment.yaml | 6 ++++++ .../file_to_blackhole_0ms_latency/lading/lading.yaml | 12 ++++++------ .../datadog-agent/conf.d/disk-listener.d/conf.yaml | 2 +- .../file_to_blackhole_1000ms_latency/experiment.yaml | 6 ++++++ .../lading/lading.yaml | 12 ++++++------ .../datadog-agent/conf.d/disk-listener.d/conf.yaml | 2 +- .../file_to_blackhole_100ms_latency/experiment.yaml | 6 ++++++ .../lading/lading.yaml | 12 ++++++------ .../datadog-agent/conf.d/disk-listener.d/conf.yaml | 2 +- .../file_to_blackhole_300ms_latency/experiment.yaml | 6 ++++++ .../lading/lading.yaml | 12 ++++++------ .../datadog-agent/conf.d/disk-listener.d/conf.yaml | 2 +- .../file_to_blackhole_500ms_latency/experiment.yaml | 6 ++++++ .../lading/lading.yaml | 12 ++++++------ 15 files changed, 65 insertions(+), 35 deletions(-) diff --git a/test/regression/cases/file_to_blackhole_0ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml b/test/regression/cases/file_to_blackhole_0ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml index 2737fc2dd4f7c1..ec51a59de1c46e 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml @@ -1,5 +1,5 @@ logs: - type: file - path: "/tmp/smp/*.log" + path: "/smp-shared/*.log" service: "my-service" source: "my-client-app" diff --git a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml index 215dcfa86cc48e..64fe41015db6f7 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml @@ -31,3 +31,9 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB + + - name: lost_bytes + description: "Allowable bytes not polled by log Agent" + bounds: + series: lost_bytes + upper_bound: 0KB diff --git a/test/regression/cases/file_to_blackhole_0ms_latency/lading/lading.yaml b/test/regression/cases/file_to_blackhole_0ms_latency/lading/lading.yaml index 68bf4bdd44bea2..e5cf494648f87e 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency/lading/lading.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency/lading/lading.yaml @@ -1,16 +1,16 @@ generator: - file_gen: - logrotate: + logrotate_fs: seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - root: "/tmp/smp" concurrent_logs: 8 - maximum_bytes_per_log: "500MiB" + maximum_bytes_per_log: 500MB total_rotations: 5 - max_depth: 1 # flat, all logs are /tmp/smp/12345.log + max_depth: 0 variant: "ascii" - bytes_per_second: "10MiB" - maximum_prebuild_cache_size_bytes: "300MiB" + bytes_per_second: 10MB + maximum_prebuild_cache_size_bytes: 300MB + mount_point: /smp-shared blackhole: - http: diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml index 2737fc2dd4f7c1..ec51a59de1c46e 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml @@ -1,5 +1,5 @@ logs: - type: file - path: "/tmp/smp/*.log" + path: "/smp-shared/*.log" service: "my-service" source: "my-client-app" diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml index 215dcfa86cc48e..64fe41015db6f7 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml @@ -31,3 +31,9 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB + + - name: lost_bytes + description: "Allowable bytes not polled by log Agent" + bounds: + series: lost_bytes + upper_bound: 0KB diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency/lading/lading.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency/lading/lading.yaml index 7cff76eb2e8181..cf3c07bf79121b 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency/lading/lading.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency/lading/lading.yaml @@ -1,16 +1,16 @@ generator: - file_gen: - logrotate: + logrotate_fs: seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - root: "/tmp/smp" concurrent_logs: 8 - maximum_bytes_per_log: "500MiB" + maximum_bytes_per_log: 500MB total_rotations: 5 - max_depth: 1 # flat, all logs are /tmp/smp/12345.log + max_depth: 0 variant: "ascii" - bytes_per_second: "10MiB" - maximum_prebuild_cache_size_bytes: "300MiB" + bytes_per_second: 10MB + maximum_prebuild_cache_size_bytes: 300MB + mount_point: /smp-shared blackhole: - http: diff --git a/test/regression/cases/file_to_blackhole_100ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml b/test/regression/cases/file_to_blackhole_100ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml index 2737fc2dd4f7c1..ec51a59de1c46e 100644 --- a/test/regression/cases/file_to_blackhole_100ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml +++ b/test/regression/cases/file_to_blackhole_100ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml @@ -1,5 +1,5 @@ logs: - type: file - path: "/tmp/smp/*.log" + path: "/smp-shared/*.log" service: "my-service" source: "my-client-app" diff --git a/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml index 215dcfa86cc48e..64fe41015db6f7 100644 --- a/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml @@ -31,3 +31,9 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB + + - name: lost_bytes + description: "Allowable bytes not polled by log Agent" + bounds: + series: lost_bytes + upper_bound: 0KB diff --git a/test/regression/cases/file_to_blackhole_100ms_latency/lading/lading.yaml b/test/regression/cases/file_to_blackhole_100ms_latency/lading/lading.yaml index 1cf21c8935cedb..d1c1648be96c1c 100644 --- a/test/regression/cases/file_to_blackhole_100ms_latency/lading/lading.yaml +++ b/test/regression/cases/file_to_blackhole_100ms_latency/lading/lading.yaml @@ -1,16 +1,16 @@ generator: - file_gen: - logrotate: + logrotate_fs: seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - root: "/tmp/smp" concurrent_logs: 8 - maximum_bytes_per_log: "500MiB" + maximum_bytes_per_log: 500MB total_rotations: 5 - max_depth: 1 # flat, all logs are /tmp/smp/12345.log + max_depth: 0 variant: "ascii" - bytes_per_second: "10MiB" - maximum_prebuild_cache_size_bytes: "300MiB" + bytes_per_second: 10MB + maximum_prebuild_cache_size_bytes: 300MB + mount_point: /smp-shared blackhole: - http: diff --git a/test/regression/cases/file_to_blackhole_300ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml b/test/regression/cases/file_to_blackhole_300ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml index 2737fc2dd4f7c1..ec51a59de1c46e 100644 --- a/test/regression/cases/file_to_blackhole_300ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml +++ b/test/regression/cases/file_to_blackhole_300ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml @@ -1,5 +1,5 @@ logs: - type: file - path: "/tmp/smp/*.log" + path: "/smp-shared/*.log" service: "my-service" source: "my-client-app" diff --git a/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml index 215dcfa86cc48e..64fe41015db6f7 100644 --- a/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml @@ -31,3 +31,9 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB + + - name: lost_bytes + description: "Allowable bytes not polled by log Agent" + bounds: + series: lost_bytes + upper_bound: 0KB diff --git a/test/regression/cases/file_to_blackhole_300ms_latency/lading/lading.yaml b/test/regression/cases/file_to_blackhole_300ms_latency/lading/lading.yaml index f8f94fbbebe2f1..dc84682becb8ab 100644 --- a/test/regression/cases/file_to_blackhole_300ms_latency/lading/lading.yaml +++ b/test/regression/cases/file_to_blackhole_300ms_latency/lading/lading.yaml @@ -1,16 +1,16 @@ generator: - file_gen: - logrotate: + logrotate_fs: seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - root: "/tmp/smp" concurrent_logs: 8 - maximum_bytes_per_log: "500MiB" + maximum_bytes_per_log: 500MB total_rotations: 5 - max_depth: 1 # flat, all logs are /tmp/smp/12345.log + max_depth: 0 variant: "ascii" - bytes_per_second: "10MiB" - maximum_prebuild_cache_size_bytes: "300MiB" + bytes_per_second: 10MB + maximum_prebuild_cache_size_bytes: 300MB + mount_point: /smp-shared blackhole: - http: diff --git a/test/regression/cases/file_to_blackhole_500ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml b/test/regression/cases/file_to_blackhole_500ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml index 2737fc2dd4f7c1..ec51a59de1c46e 100644 --- a/test/regression/cases/file_to_blackhole_500ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml +++ b/test/regression/cases/file_to_blackhole_500ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml @@ -1,5 +1,5 @@ logs: - type: file - path: "/tmp/smp/*.log" + path: "/smp-shared/*.log" service: "my-service" source: "my-client-app" diff --git a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml index 215dcfa86cc48e..64fe41015db6f7 100644 --- a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml @@ -31,3 +31,9 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB + + - name: lost_bytes + description: "Allowable bytes not polled by log Agent" + bounds: + series: lost_bytes + upper_bound: 0KB diff --git a/test/regression/cases/file_to_blackhole_500ms_latency/lading/lading.yaml b/test/regression/cases/file_to_blackhole_500ms_latency/lading/lading.yaml index d88c5c4679fcfd..90920efb161d42 100644 --- a/test/regression/cases/file_to_blackhole_500ms_latency/lading/lading.yaml +++ b/test/regression/cases/file_to_blackhole_500ms_latency/lading/lading.yaml @@ -1,16 +1,16 @@ generator: - file_gen: - logrotate: + logrotate_fs: seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - root: "/tmp/smp" concurrent_logs: 8 - maximum_bytes_per_log: "500MiB" + maximum_bytes_per_log: 500MB total_rotations: 5 - max_depth: 1 # flat, all logs are /tmp/smp/12345.log + max_depth: 0 variant: "ascii" - bytes_per_second: "10MiB" - maximum_prebuild_cache_size_bytes: "300MiB" + bytes_per_second: 10MB + maximum_prebuild_cache_size_bytes: 300MB + mount_point: /smp-shared blackhole: - http: From c8e532ccbc2e22a5762c236d0276939beaaa4808 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 14:05:56 -0800 Subject: [PATCH 2/3] only make lost_bytes claim on 100ms and less Signed-off-by: Brian L. Troutwine --- .../cases/file_to_blackhole_1000ms_latency/experiment.yaml | 6 ------ .../cases/file_to_blackhole_300ms_latency/experiment.yaml | 6 ------ .../cases/file_to_blackhole_500ms_latency/experiment.yaml | 6 ------ 3 files changed, 18 deletions(-) diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml index 64fe41015db6f7..215dcfa86cc48e 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml @@ -31,9 +31,3 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB - - - name: lost_bytes - description: "Allowable bytes not polled by log Agent" - bounds: - series: lost_bytes - upper_bound: 0KB diff --git a/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml index 64fe41015db6f7..215dcfa86cc48e 100644 --- a/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml @@ -31,9 +31,3 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB - - - name: lost_bytes - description: "Allowable bytes not polled by log Agent" - bounds: - series: lost_bytes - upper_bound: 0KB diff --git a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml index 64fe41015db6f7..215dcfa86cc48e 100644 --- a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml @@ -31,9 +31,3 @@ checks: series: total_rss_bytes # The machine has 12GiB free. upper_bound: 1.2GiB - - - name: lost_bytes - description: "Allowable bytes not polled by log Agent" - bounds: - series: lost_bytes - upper_bound: 0KB From a7e507ec652c772b5bd3333e2f495321686d1a07 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 14:45:54 -0800 Subject: [PATCH 3/3] Update test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml Co-authored-by: Scott Opell --- .../cases/file_to_blackhole_0ms_latency/experiment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml index 64fe41015db6f7..4884b1e7a29646 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml @@ -33,7 +33,7 @@ checks: upper_bound: 1.2GiB - name: lost_bytes - description: "Allowable bytes not polled by log Agent" + description: "Available bytes not polled by log Agent" bounds: series: lost_bytes upper_bound: 0KB