From 3e3df17de544bc50586aa8167c2b062bb5d8d852 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Thu, 17 Aug 2023 11:52:50 +0200 Subject: [PATCH] chunk: add option-destination-threaded-batching It includes the batch-lines() and batch-timeout() options, which always come hand-in-hand. Some places forgot to include batch-timeout(), but those drivers have that option. Including option-destination-threaded-batching takes care of this problem. Signed-off-by: Attila Szakacs --- .../reference-destination-amqp/_index.md | 4 +--- .../reference-destination-http-nonjava/_index.md | 10 +--------- .../reference-destination-kafka-c/_index.md | 8 +------- .../reference-destination-mongodb/_index.md | 4 +--- .../reference-destination-redis/_index.md | 7 ++----- .../reference-destination-riemann/_index.md | 4 +--- .../reference-destination-smtp/_index.md | 4 +--- .../reference-destination-sql/_index.md | 4 +--- .../reference-destination-stomp/_index.md | 4 +--- .../reference-destination-discord/_index.md | 9 +-------- .../reference-destination-slack/_index.md | 9 +-------- .../destination-sumologic-options-http/_index.md | 2 +- .../chunk/option-destination-threaded-batching.md | 6 ++++++ .../wnt/n-options-only-effective-for-sync-send(yes).md | 2 +- 14 files changed, 20 insertions(+), 57 deletions(-) create mode 100644 content/headless/chunk/option-destination-threaded-batching.md diff --git a/content/chapter-destinations/configuring-destinations-amqp/reference-destination-amqp/_index.md b/content/chapter-destinations/configuring-destinations-amqp/reference-destination-amqp/_index.md index 33f0b1a3..bea6429a 100644 --- a/content/chapter-destinations/configuring-destinations-amqp/reference-destination-amqp/_index.md +++ b/content/chapter-destinations/configuring-destinations-amqp/reference-destination-amqp/_index.md @@ -25,9 +25,7 @@ The `amqp()` destination has the following options: {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} ## body() {#amqp-option-body} diff --git a/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md b/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md index 4d25daf1..03a0a805 100644 --- a/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md +++ b/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md @@ -12,19 +12,11 @@ The `http` destination of {{% param "product.abbrev" %}} can directly post log m For details on how this option influences HTTP batch mode, see [http: Posting messages over HTTP without Java]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/_index.md" >}}) - -{{% include-headless "chunk/option-destination-batch-lines.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} For details on how this option influences HTTP batch mode, see [http: Posting messages over HTTP without Java]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/_index.md" >}}) - -{{% include-headless "chunk/option-d-b-t-http-spl-sent-pub.md" %}} - -For details on how this option influences HTTP batch mode, see [http: Posting messages over HTTP without Java]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/_index.md" >}}) - - - ## body() | | | diff --git a/content/chapter-destinations/configuring-destinations-kafka-c/reference-destination-kafka-c/_index.md b/content/chapter-destinations/configuring-destinations-kafka-c/reference-destination-kafka-c/_index.md index ced883f9..5a93094d 100644 --- a/content/chapter-destinations/configuring-destinations-kafka-c/reference-destination-kafka-c/_index.md +++ b/content/chapter-destinations/configuring-destinations-kafka-c/reference-destination-kafka-c/_index.md @@ -18,7 +18,7 @@ The following options are required: `bootstrap-servers()`, `topic()`. Note that {{< include-headless "chunk/kafka-c-impl-required-options-note.md" >}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} {{< include-headless "wnt/n-options-only-effective-for-sync-send(yes).md" >}} @@ -28,12 +28,6 @@ If you set `sync-send()` to `"yes"`, the number you specify for `batch-lines()` {{% /alert %}} - - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} - -{{< include-headless "wnt/n-options-only-effective-for-sync-send(yes).md" >}} - {{% alert title="Note" color="info" %}} When setting `batch-timeout()`, consider the value of the `transaction.timeout.ms` Kafka property. If in case of timeout (that is, if {{% param "product.abbrev" %}} does not receive `batch-lines()` amount of messages) the value of `batch-timeout()` exceeds the value of `transaction.timeout.ms`, {{% param "product.abbrev" %}} will not send out messages in time. diff --git a/content/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/_index.md b/content/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/_index.md index 5ee4c2ac..130811fa 100644 --- a/content/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/_index.md +++ b/content/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/_index.md @@ -63,9 +63,7 @@ Hazard of data loss! The {{% param "product.abbrev" %}} application does not ver {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} {{< include-headless "chunk/option-destination-frac-digits.md" >}} diff --git a/content/chapter-destinations/configuring-destinations-redis/reference-destination-redis/_index.md b/content/chapter-destinations/configuring-destinations-redis/reference-destination-redis/_index.md index ed73379f..93bb5cfc 100644 --- a/content/chapter-destinations/configuring-destinations-redis/reference-destination-redis/_index.md +++ b/content/chapter-destinations/configuring-destinations-redis/reference-destination-redis/_index.md @@ -17,9 +17,8 @@ The `redis()` destination has the following options: *Description:* The password used for authentication on a password-protected Redis server. Available in {{% param "product.abbrev" %}} version 3.10 and later. -{{% include-headless "chunk/option-destination-batch-lines.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} ## command() {#redis-option-command} @@ -42,9 +41,7 @@ Note the following points when using the `redis()` destination: {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} {{< include-headless "chunk/option-destination-hook.md" >}} diff --git a/content/chapter-destinations/configuring-destinations-riemann/reference-destination-riemann/_index.md b/content/chapter-destinations/configuring-destinations-riemann/reference-destination-riemann/_index.md index d8976a3d..8071200b 100644 --- a/content/chapter-destinations/configuring-destinations-riemann/reference-destination-riemann/_index.md +++ b/content/chapter-destinations/configuring-destinations-riemann/reference-destination-riemann/_index.md @@ -97,13 +97,11 @@ Note that the time format parameter requires: {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} If an error occurs while sending the messages to the server, {{% param "product.abbrev" %}} will try to resend every message from the batch. If it does not succeed (you can set the number of retry attempts in the `retries()` option), {{% param "product.abbrev" %}} drops every message in the batch. -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} - {{< include-headless "chunk/option-destination-hook.md" >}} ## host() {#riemann-option-host} diff --git a/content/chapter-destinations/configuring-destinations-smtp/reference-destination-smtp/_index.md b/content/chapter-destinations/configuring-destinations-smtp/reference-destination-smtp/_index.md index 5ef025f1..aff9596e 100644 --- a/content/chapter-destinations/configuring-destinations-smtp/reference-destination-smtp/_index.md +++ b/content/chapter-destinations/configuring-destinations-smtp/reference-destination-smtp/_index.md @@ -16,9 +16,7 @@ The `smtp()` sends email messages using SMTP, without needing external applicati {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} {{% include-headless "chunk/smtp-option-from.md" %}} diff --git a/content/chapter-destinations/configuring-destinations-sql/reference-destination-sql/_index.md b/content/chapter-destinations/configuring-destinations-sql/reference-destination-sql/_index.md index 759321c2..afb575d4 100644 --- a/content/chapter-destinations/configuring-destinations-sql/reference-destination-sql/_index.md +++ b/content/chapter-destinations/configuring-destinations-sql/reference-destination-sql/_index.md @@ -8,9 +8,7 @@ This driver sends messages into an SQL database. The `sql()` destination has the {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} ## columns() {#sql-option-columns} diff --git a/content/chapter-destinations/configuring-destinations-stomp/reference-destination-stomp/_index.md b/content/chapter-destinations/configuring-destinations-stomp/reference-destination-stomp/_index.md index a4c16447..b7fc1bab 100644 --- a/content/chapter-destinations/configuring-destinations-stomp/reference-destination-stomp/_index.md +++ b/content/chapter-destinations/configuring-destinations-stomp/reference-destination-stomp/_index.md @@ -39,9 +39,7 @@ The `stomp()` destination has the following options: {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} {{< include-headless "chunk/option-destination-hook.md" >}} diff --git a/content/chapter-destinations/destination-discord/reference-destination-discord/_index.md b/content/chapter-destinations/destination-discord/reference-destination-discord/_index.md index 13463282..9ae68ab6 100644 --- a/content/chapter-destinations/destination-discord/reference-destination-discord/_index.md +++ b/content/chapter-destinations/destination-discord/reference-destination-discord/_index.md @@ -24,18 +24,11 @@ For details on how this option influences HTTP batch mode, see [http: Posting me -{{% include-headless "chunk/option-destination-batch-lines.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} For details on how this option influences HTTP batch mode, see [http: Posting messages over HTTP without Java]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/_index.md" >}}) - -{{% include-headless "chunk/option-d-b-t-http-spl-sent-pub.md" %}} - -For details on how this option influences HTTP batch mode, see [http: Posting messages over HTTP without Java]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/_index.md" >}}) - - - {{% include-headless "chunk/option-destination-tls-ca-dir.md" %}} {{% include-headless "chunk/topic-tls-block-http.md" %}} diff --git a/content/chapter-destinations/destination-slack/reference-destination-slack/_index.md b/content/chapter-destinations/destination-slack/reference-destination-slack/_index.md index dd7c6988..cb710335 100644 --- a/content/chapter-destinations/destination-slack/reference-destination-slack/_index.md +++ b/content/chapter-destinations/destination-slack/reference-destination-slack/_index.md @@ -46,18 +46,11 @@ For details on how this option influences batch mode, see [http: Posting message -{{% include-headless "chunk/option-destination-batch-lines.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} For details on how this option influences batch mode, see [http: Posting messages over HTTP without Java]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/_index.md" >}}) - -{{% include-headless "chunk/option-destination-batch-timeout.md" %}} - -For details on how this option influences batch mode, see [http: Posting messages over HTTP without Java]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/_index.md" >}}) - - - {{% include-headless "chunk/option-destination-tls-ca-dir.md" %}} {{% include-headless "chunk/topic-tls-block-http.md" %}} diff --git a/content/chapter-destinations/destination-sumologic-intro/destination-sumologic-options/destination-sumologic-options-http/_index.md b/content/chapter-destinations/destination-sumologic-intro/destination-sumologic-options/destination-sumologic-options-http/_index.md index 6d91362f..4e282bca 100644 --- a/content/chapter-destinations/destination-sumologic-intro/destination-sumologic-options/destination-sumologic-options-http/_index.md +++ b/content/chapter-destinations/destination-sumologic-intro/destination-sumologic-options/destination-sumologic-options-http/_index.md @@ -10,7 +10,7 @@ In addition, the `sumologic-http()` destination also has the following options. {{% include-headless "chunk/option-destination-batch-bytes.md" %}} -{{% include-headless "chunk/option-destination-batch-lines.md" %}} +{{% include-headless "chunk/option-destination-threaded-batching.md" %}} {{% include-headless "chunk/option-destination-tls-ca-dir.md" %}} diff --git a/content/headless/chunk/option-destination-threaded-batching.md b/content/headless/chunk/option-destination-threaded-batching.md new file mode 100644 index 00000000..e981f74b --- /dev/null +++ b/content/headless/chunk/option-destination-threaded-batching.md @@ -0,0 +1,6 @@ +--- +--- + +{{% include-headless "chunk/option-destination-batch-lines.md" %}} + +{{% include-headless "chunk/option-destination-batch-timeout.md" %}} diff --git a/content/headless/wnt/n-options-only-effective-for-sync-send(yes).md b/content/headless/wnt/n-options-only-effective-for-sync-send(yes).md index 0de7e0f3..6917b690 100644 --- a/content/headless/wnt/n-options-only-effective-for-sync-send(yes).md +++ b/content/headless/wnt/n-options-only-effective-for-sync-send(yes).md @@ -3,6 +3,6 @@ {{% alert title="Note" color="info" %}} -The {{% param "product.abbrev" %}} configuration accepts this option with `sync-send()` set to both `"yes"` or `"no"`, but the option will only take effect if you set `sync-send()` to `"yes"`. +The {{% param "product.abbrev" %}} configuration accepts `batch-lines()` with `sync-send()` set to both `"yes"` or `"no"`, but the option will only take effect if you set `sync-send()` to `"yes"`. {{% /alert %}}