#minor Add an option to send request only once per batch

.github/workflows/build_and_test.yaml

@@ -95,6 +95,7 @@ jobs:
           wait_for_pod_ready "sidecar-pythonscript"
           wait_for_pod_ready "sidecar-pythonscript-logfile"
           wait_for_pod_ready "sidecar-logtofile-pythonscript"
+          wait_for_pod_ready "sidecar-req-once-per-batch"
           wait_for_pod_ready "dummy-server-pod"
 
       - name: Install Configmaps and Secrets
@@ -107,7 +108,7 @@ jobs:
           sleep 20
           echo "Installing resources..."
           kubectl apply -f "test/resources/resources.yaml"
-          pods=("sidecar" "sidecar-basicauth-args" "sidecar-5xx" "sidecar-pythonscript" "sidecar-pythonscript-logfile")
+          pods=("sidecar" "sidecar-basicauth-args" "sidecar-5xx" "sidecar-pythonscript" "sidecar-pythonscript-logfile" "sidecar-req-once-per-batch")
           resources=("sample-configmap" "sample-secret-binary" "absolute-configmap" "relative-configmap" "change-dir-configmap" "similar-configmap-secret" "url-configmap-500" "url-configmap-basic-auth" "sample-configmap")
           for p in ${pods[*]}; do
             for r in ${resources[*]}; do
@@ -125,6 +126,10 @@ jobs:
           kubectl logs sidecar-pythonscript > /tmp/logs/sidecar-pythonscript.log
           kubectl logs sidecar-pythonscript-logfile > /tmp/logs/sidecar-pythonscript-logfile.log
           kubectl logs dummy-server-pod > /tmp/logs/dummy-server.log
+
+          # Sleep more to pass WATCH_SERVER_TIMEOUT seconds for request once per batch to occur
+          sleep 40
+          kubectl logs sidecar-req-once-per-batch > /tmp/logs/sidecar-req-once-per-batch.log
       - name: Upload artifacts (pod logs)
         uses: actions/upload-artifact@v4
         with:
@@ -244,4 +249,8 @@ jobs:
           kubectl exec sidecar -- sh -c "! test -e /tmp/relative/relative.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/relative/change-relative.txt" &&
           kubectl exec sidecar -- sh -c "! test -e /tmp/orig-dir/change-dir.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/new-dir/change-dir.txt" &&
           kubectl exec sidecar -- sh -c "! test -e /tmp/similar-configmap.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-configmap.txt" &&
-          kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt"
+          kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt"
+      - name: Verify sidecar-req-once-per-batch logs after initial sync
+        run: |
+          test $(cat /tmp/logs/sidecar-req-once-per-batch.log | grep "Request once per batch will be enabled" | wc -l) = "1" &&
+          test $(cat /tmp/logs/sidecar-req-once-per-batch.log | grep "Starting batch request" | wc -l) = "2"

README.md

@@ -39,7 +39,7 @@ All are identical multi-arch images built for `amd64`, `arm64`, `arm/v7`, `ppc64
   - Values can also be base64 encoded URLs that download binary data e.g. executables
     - The key in the `ConfigMap`/`Secret` must end with "`.url`" ([see](https://github.com/kiwigrid/k8s-sidecar/blob/master/test/resources/resources.yaml#L84))
 
-# Usage 
+# Usage
 
 Example for a simple deployment can be found in [`example.yaml`](./examples/example.yaml). Depending on the cluster setup you have to grant yourself admin rights first:
 ```shell
@@ -84,6 +84,7 @@ If the filename ends with `.url` suffix, the content will be processed as a URL
 | `REQ_USERNAME`             | Username to use for basic authentication for requests to `REQ_URL` and for `*.url` triggered requests                                                                                                                                                                                                                               | false    | -                                         | string  |
 | `REQ_PASSWORD`             | Password to use for basic authentication for requests to `REQ_URL` and for `*.url` triggered requests                                                                                                                                                                                                                               | false    | -                                         | string  |
 | `REQ_BASIC_AUTH_ENCODING`  | Which encoding to use for username and password as [by default it's undefined](https://datatracker.ietf.org/doc/html/rfc7617) (e.g. `utf-8`).                                                                                                                                                                                       | false    | `latin1`                                  | string  |
+| `REQ_ONCE_PER_BATCH`       | Send request to `REQ_URL` only once each `WATCH_SERVER_TIMEOUT`. By default request is made after each change. Applicable only to `METHOD=WATCH`. | false | - | boolean |
 | `SCRIPT`                   | Absolute path to a script to execute after a configmap got reloaded. It runs before calls to `REQ_URI`. If the file is not executable it will be passed to `sh`. Otherwise it's executed as is. [Shebangs](https://en.wikipedia.org/wiki/Shebang_(Unix)) known to work are `#!/bin/sh` and `#!/usr/bin/env python`                  | false    | -                                         | string  |
 | `ERROR_THROTTLE_SLEEP`     | How many seconds to wait before watching resources again when an error occurs                                                                                                                                                                                                                                                       | false    | `5`                                       | integer |
 | `SKIP_TLS_VERIFY`          | Set to `true` to skip tls verification for kube api calls                                                                                                                                                                                                                                                                           | false    | -                                         | boolean |

src/resources.py

@@ -305,7 +305,7 @@ def _update_file(data_key, data_content, dest_folder, metadata, resource,
 
 def _watch_resource_iterator(label, label_value, target_folder, request_url, request_method, request_payload,
                              namespace, folder_annotation, resource, unique_filenames, script, enable_5xx,
-                             ignore_already_processed):
+                             ignore_already_processed, request_once_per_batch):
     v1 = client.CoreV1Api()
     # Filter resources based on label and value or just label
     label_selector = f"{label}={label_value}" if label_value else label
@@ -322,6 +322,9 @@ def _watch_resource_iterator(label, label_value, target_folder, request_url, req
 
     stream = watch.Watch().stream(getattr(v1, _list_namespace[namespace][resource]), **additional_args)
 
+    # Used if request_once_per_batch is enabled
+    any_files_changed = False
+
     # Process events
     for event in stream:
         item = event['object']
@@ -355,12 +358,18 @@ def _watch_resource_iterator(label, label_value, target_folder, request_url, req
         else:
             files_changed |= _process_secret(dest_folder, item, resource, unique_filenames, enable_5xx, item_removed)
 
+        any_files_changed |= files_changed
+
         if script and files_changed:
             execute(script)
 
-        if request_url and files_changed:
+        if request_url and files_changed and not request_once_per_batch:
             request(request_url, request_method, enable_5xx, request_payload)
 
+    if request_url and any_files_changed and request_once_per_batch:
+        logger.debug(f"Starting batch request")
+        request(request_url, request_method, enable_5xx, request_payload)
+
 
 def _watch_resource_loop(mode, *args):
     while True:
@@ -390,11 +399,11 @@ def _watch_resource_loop(mode, *args):
 
 def watch_for_changes(mode, label, label_value, target_folder, request_url, request_method, request_payload,
                       current_namespace, folder_annotation, resources, unique_filenames, script, enable_5xx,
-                      ignore_already_processed):
+                      ignore_already_processed, request_once_per_batch):
     processes = _start_watcher_processes(current_namespace, folder_annotation, label,
                                          label_value, request_method, mode, request_payload, resources,
                                          target_folder, unique_filenames, script, request_url, enable_5xx,
-                                         ignore_already_processed)
+                                         ignore_already_processed, request_once_per_batch)
 
     while True:
         died = False
@@ -414,14 +423,14 @@ def watch_for_changes(mode, label, label_value, target_folder, request_url, requ
 
 def _start_watcher_processes(namespace, folder_annotation, label, label_value, request_method,
                              mode, request_payload, resources, target_folder, unique_filenames, script, request_url,
-                             enable_5xx, ignore_already_processed):
+                             enable_5xx, ignore_already_processed, request_once_per_batch):
     processes = []
     for resource in resources:
         for ns in namespace.split(','):
             proc = Process(target=_watch_resource_loop,
                            args=(mode, label, label_value, target_folder, request_url, request_method, request_payload,
                                  ns, folder_annotation, resource, unique_filenames, script, enable_5xx,
-                                 ignore_already_processed)
+                                 ignore_already_processed, request_once_per_batch)
                            )
             proc.daemon = True
             proc.start()

src/sidecar.py

@@ -24,6 +24,7 @@
 REQ_PAYLOAD = "REQ_PAYLOAD"
 REQ_URL = "REQ_URL"
 REQ_METHOD = "REQ_METHOD"
+REQ_ONCE_PER_BATCH = "REQ_ONCE_PER_BATCH"
 SCRIPT = "SCRIPT"
 ENABLE_5XX = "ENABLE_5XX"
 IGNORE_ALREADY_PROCESSED = "IGNORE_ALREADY_PROCESSED"
@@ -72,10 +73,19 @@ def main():
 
     request_method = os.getenv(REQ_METHOD)
     request_url = os.getenv(REQ_URL)
-   
+
     request_payload = os.getenv(REQ_PAYLOAD)
     if request_payload:
         request_payload = prepare_payload(os.getenv(REQ_PAYLOAD))
+
+    request_once_per_batch = os.getenv(REQ_ONCE_PER_BATCH)
+    if request_once_per_batch is not None and request_once_per_batch.lower() == "true":
+        logger.info(f"Request once per batch will be enabled.")
+        request_once_per_batch = True
+    else:
+        logger.info(f"Request once per batch will not be enabled.")
+        request_once_per_batch = False
+
     script = os.getenv(SCRIPT)
 
     _initialize_kubeclient_configuration()
@@ -131,7 +141,8 @@ def main():
     else:
         watch_for_changes(method, label, label_value, target_folder, request_url, request_method, request_payload,
                           namespace, folder_annotation, resources, unique_filenames, script, enable_5xx,
-                          ignore_already_processed)
+                          ignore_already_processed,
+                          request_once_per_batch)
 
 
 def _initialize_kubeclient_configuration():

test/resources/sidecar.yaml

@@ -168,7 +168,7 @@ data:
     root:
         level: DEBUG
         handlers: [console]
-        
+
     handlers:
         console:
             class: logging.StreamHandler
@@ -180,7 +180,7 @@ data:
             (): logger.JsonFormatter
             format: '%(levelname)s %(message)s'
             rename_fields: {
-                    "message": "msg", 
+                    "message": "msg",
                     "levelname": "level"
                     }
 ---
@@ -388,4 +388,37 @@ metadata:
 type: Opaque
 stringData:
   username: "user1"
-  password: "abcdefghijklmnopqrstuvwxyz"
+  password: "abcdefghijklmnopqrstuvwxyz"
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sidecar-req-once-per-batch
+  namespace: default
+spec:
+  serviceAccountName: sample-acc
+  containers:
+  - name: sidecar
+    image: kiwigrid/k8s-sidecar:testing
+    volumeMounts:
+    - name: shared-volume
+      mountPath: /tmp/
+    env:
+      - name: LABEL
+        value: "findme"
+      - name: FOLDER
+        value: /tmp/
+      - name: RESOURCE
+        value: both
+      - name: REQ_URL
+        value: http://dummy-server/
+      - name: REQ_METHOD
+        value: GET
+      - name: REQ_ONCE_PER_BATCH
+        value: "true"
+      - name: LOG_LEVEL
+        value: "DEBUG"
+  volumes:
+  - name: shared-volume
+    emptyDir: {}

test/server/server.py

@@ -45,4 +45,5 @@ async def read_secure_data(auth: HTTPBasicCredentials = Depends(basic_auth_schem
             detail=f"Incorrect user (${auth.username}) or password (${auth.password})",
             headers={"WWW-Authenticate": "Basic"},
         )
-    return 'allowed'
+    return 'allowed'
+