Add @timestamp and delivery_timestamp fields to the commands inde…

…x data model (#568) * Add ECS fields @timestamp and delivery_timestamp to command index data model * Update command event_generator with new fields * Move delivery_timestamp to doc level Update command event_generator Remove delivery_timestamp from custom command fields * Move delivery_timestamp definition to a new file * Rename delivery_timestamp custom group to 'base' Now delivery_timestamp can be used as part of base Updated the command subset.yml * Fix the include_all_fields option on the event_generator * Update command event_generator Remove duplicated fields and order the full document Update command ECS index documentation * Update event_generator to use date_time_no_millis compatible datetime format * Improve command.timeout description * Add new revision to commands.md --------- Co-authored-by: Álex Ruiz <alejandro.ruiz.becerra@wazuh.com>
wazuh · Dec 3, 2024 · e626d3d · e626d3d
1 parent b9a8ff3
commit e626d3d
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 18 deletions.
diff --git a/ecs/command/event-generator/event_generator.py b/ecs/command/event-generator/event_generator.py
@@ -1,6 +1,7 @@
 #!/bin/python3
 
 import argparse
+import datetime
 import json
 import logging
 import random
@@ -10,6 +11,7 @@
 
 LOG_FILE = 'generate_data.log'
 GENERATED_DATA_FILE = 'generatedData.json'
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 # Default values
 INDEX_NAME = ".commands"
 USERNAME = "admin"
@@ -24,36 +26,49 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
+def generate_random_date(initial_date=None, days_range=30):
+    if initial_date is None:
+        initial_date = datetime.datetime.now(datetime.timezone.utc)
+    random_days = random.randint(0, days_range)
+    new_timestamp = initial_date + datetime.timedelta(days=random_days)
+    return new_timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
 def generate_random_command(include_all_fields=False):
-    document = {
+    command = {
         "source": random.choice(["Users/Services", "Engine", "Content manager"]),
         "user": f"user{random.randint(1, 100)}",
         "target": {
             "id": f"target{random.randint(1, 10)}",
             "type": random.choice(["agent", "group", "server"])
         },
         "action": {
-            "name": random.choice(["restart", "update", "change_group", "apply_policy"]),
+            "name": random.choice(["restart", "update","change_group", "apply_policy"]),
             "args": [f"/path/to/executable/arg{random.randint(1, 10)}"],
             "version": f"v{random.randint(1, 5)}"
         },
         "timeout": random.randint(10, 100)
     }
-
     if include_all_fields:
-        document["agent"]["groups"] = [f"group{random.randint(1, 5)}"],
-        document["command"]["status"] = random.choice(
-            ["pending", "sent", "success", "failure"])
-        document["command"]["result"] = {
-            "code": random.randint(0, 255),
-            "message": f"Result message {random.randint(1, 1000)}",
-            "data": f"Result data {random.randint(1, 100)}"
+        document = {
+            "@timestamp": generate_random_date(),
+            "delivery_timestamp": generate_random_date(),
+            "agent": {"groups": [f"group{random.randint(1, 5)}"]},
+            "command": {
+                **command,
+                "status": random.choice(["pending", "sent", "success", "failure"]),
+                "result": {
+                    "code": random.randint(0, 255),
+                    "message": f"Result message {random.randint(1, 1000)}",
+                    "data": f"Result data {random.randint(1, 100)}"
+                },
+                "request_id": str(uuid.uuid4()),
+                "order_id": str(uuid.uuid4())
+            }
         }
-        # Generate UUIDs for request_id and order_id
-        document["command"]["request_id"] = str(uuid.uuid4())
-        document["command"]["order_id"] = str(uuid.uuid4())
+        return document
 
-    return document
+    return command
 
 
 def generate_random_data(number, include_all_fields=False):
@@ -74,10 +89,10 @@ def inject_events(ip, port, index, username, password, data, use_index=False):
             if use_index:
                 # Generate UUIDs for the document id
                 doc_id = str(uuid.uuid4())
-                url = f'https://{ip}:{port}/{index}/_doc/{doc_id}'
+                url = f'http://{ip}:{port}/{index}/_doc/{doc_id}'
             else:
                 # Default URL for command manager API without the index
-                url = f'https://{ip}:{port}/_plugins/_command_manager/commands'
+                url = f'http://{ip}:{port}/_plugins/_command_manager/commands'
             response = session.post(url, json=event_data, headers=headers)
             if response.status_code != 201:
                 logging.error(f'Error: {response.status_code}')

diff --git a/ecs/command/fields/custom/base.yml b/ecs/command/fields/custom/base.yml
@@ -0,0 +1,9 @@
+- name: base
+  title: Wazuh base fields
+  root: true
+  fields:
+    - name: delivery_timestamp
+      type: date
+      level: custom
+      description: >
+        The latest date-time for the command to be delivered. Calculated as the current timestamp plus the timeout.
diff --git a/ecs/command/fields/custom/command.yml b/ecs/command/fields/custom/command.yml
@@ -46,7 +46,7 @@
       type: short
       level: custom
       description: >
-        Time window in which the command has to be sent to its target.
+        Seconds in which the command has to be sent to its target.
     - name: status
       type: keyword
       level: custom

diff --git a/ecs/command/fields/subset.yml b/ecs/command/fields/subset.yml
@@ -4,6 +4,8 @@ fields:
   base:
     fields:
       tags: []
+      "@timestamp": {}
+      "delivery_timestamp": {}
   agent:
     fields:
       groups: {}

diff --git a/ecs/docs/commands.md b/ecs/docs/commands.md
@@ -1,10 +1,11 @@
 ## `commands` index data model
 
-> [!NOTE]  
+> [!NOTE]
 > rev 0.1 - September 18th, 2024: Add initial model.
 > rev 0.2 - September 30th, 2024: Change type of `request_id`, `order_id` and `id` to keyword.
 > rev 0.3 - October 3rd, 2024: Change descriptions for `command.type`, `command.action.type`, `command.request_id`, `command.order_id`.
 > rev 0.4 - October 9th, 2024: Apply changes described in https://github.com/wazuh/wazuh-indexer-plugins/issues/96#issue-2576028654.
+> rev 0.5 - December 3rd, 2024: Added `@timestamp` and `delivery_timestamp` date fields. 
 
 ### Fields summary
 
@@ -39,6 +40,8 @@ fields:
   base:
     fields:
       tags: []
+      "@timestamp": {}
+      "delivery_timestamp": {}
   agent:
     fields:
       groups: {}
@@ -127,6 +130,17 @@ fields:
       description: >
         UUID generated by the Command Manager.
 ```
+```yml
+- name: base
+  title: Wazuh base fields
+  root: true
+  fields:
+    - name: delivery_timestamp
+      type: date
+      level: custom
+      description: >
+        The latest date-time for the command to be delivered. Calculated as the current timestamp plus the timeout.
+```
 
 ### Index settings