From 883d9e3a6c46e0baba06461c09cf0443a4142e6b Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Sat, 14 Sep 2024 23:07:23 +0200 Subject: [PATCH] ci: elasticsearch: Upload script index map examples Add several examples for `upload_test_results_es.py` script usage with ElasticSearch index map files for the following use cases: * Twister test results. * Twister test results with recordings. * Memory Footprint data (`twister-footprint.json`). Signed-off-by: Dmitrii Golovanov --- scripts/ci/es_upload/README.md | 156 ++++++++++++++ .../zephyr_twister_flat_footprint_index.json | 158 ++++++++++++++ .../zephyr_twister_flat_recording_index.json | 195 ++++++++++++++++++ ..._twister_flat_recording_metrics_index.json | 150 ++++++++++++++ .../ci/es_upload/zephyr_twister_index.json | 149 +++++++++++++ 5 files changed, 808 insertions(+) create mode 100644 scripts/ci/es_upload/README.md create mode 100644 scripts/ci/es_upload/zephyr_twister_flat_footprint_index.json create mode 100644 scripts/ci/es_upload/zephyr_twister_flat_recording_index.json create mode 100644 scripts/ci/es_upload/zephyr_twister_flat_recording_metrics_index.json create mode 100644 scripts/ci/es_upload/zephyr_twister_index.json diff --git a/scripts/ci/es_upload/README.md b/scripts/ci/es_upload/README.md new file mode 100644 index 00000000000000..eb502eb63dfd69 --- /dev/null +++ b/scripts/ci/es_upload/README.md @@ -0,0 +1,156 @@ +# ElasticSearch index map examples + +This directory contains [ElasticSearch data store](https://github.com/elastic/elasticsearch) +index map examples for use by [upload_test_results_es.py](../upload_test_results_es.py) +script for [Twister](https://docs.zephyrproject.org/latest/develop/test/twister.html) +JSON reports upload. + +An index map file allows to create destination index files on ElasticSearch server +with explicit scheme having all required Twister JSON report objects associated +to proper data types, eventually to store the expected document structure. +Besides, it allows to track changes in Twister JSON report scheme +and the corresponding data scheme in the same source code repository. +An alternative way is to have index files created at the ElasticSearch server +by other tools, or rely on default data mapping with potential type misalignments. + +The command examples below are simplified with parameters essential for data transformations. +For other command line options and more details run the upload script with `--help`, +or read its source code with inline comments there. + +Use these examples as a reference for your setup. Always pay attention to possible changes +in the current Twister report format and in the upload script itself. +Tune resulting data scheme and size depending on your particular needs. + +It is recommended to try the upload script with `--dry-run` option first +to check the resulting data without its actual upload. + +You should set environment variables `ELASTICSEARCH_SERVER` and `ELASTICSEARCH_KEY` +to connect with your server running the upload script. + + +## Index creation with a map file + +Execute the upload script once for the first time to create your index at the ElasticSearch +server with the appropriate map file, for example: +```bash +python3 ./scripts/ci/upload_test_results_es.py --create-index \ + --index YOUR_INDEX_NAME \ + --map-file YOUR_INDEX_MAP.json +``` + + +## Upload transformations + +The upload script has several command line options to change `twister.json` data: +exclude excess data, extract substrings by regular expressions, change data structure +making it suitable for Kibana default visual components +(see `--help` for more details): + +* `--exclude` removes excess testsuite properties not needed to store them + at Elasticsearch index. + +* `--transform` applies regexp group parsing rules to string properties extracting + derived object properties. + +* `--flatten` changes testsuite data structure in regard of one of its list components: + either `testcases` or `recording`, so each item there becomes an independent data record + inheriting all other testsuite properties, whereas the children object's properties + are renamed with the parent object's name as a prefix: + `testcases_` or `recording_` respectively. + Only one testsuite property object can be flattened this way per index upload. + All other testsuite objects will be treated according to the index structure. + +* Other command line options: `--flatten-dict-name`, `--flatten-list-names`, +`--flatten-separator`, `--transpose-separator`, `--escape-separator`. + + +## Index map examples and use cases + + +### Twister test results + +Create the index: +```bash +python3 ./scripts/ci/upload_test_results_es.py --create-index \ + --index zephyr-test-example \ + --map-file zephyr_twister_index.json +``` + +Upload Twister test results 'as-is', without additional transformations: +```bash +python3 ./scripts/ci/upload_test_results_es.py \ + --index zephyr-test-example \ + ./twister-out/**/twister.json +``` + + +### Twister tests with recording + +Store test results with `recording` data entries, for example from +[Kernel Timer Behavior](https://github.com/zephyrproject-rtos/zephyr/tree/main/tests/kernel/timer/timer_behavior) +test suite. + +Create the index: +```bash +python3 ./scripts/ci/upload_test_results_es.py --create-index \ + --index zephyr-test-recording-example-1 \ + --map-file zephyr_twister_flat_recording_metrics_index.json +``` + +Upload data with 'flattened' test suites creating documents for each `recording` data entry. +```bash +python3 ./scripts/ci/upload_test_results_es.py \ + --index zephyr-test-recording-example-1 \ + --exclude path run_id \ + --flatten recording \ + ./twister-out/**/twister.json +``` + + +### Twister test with recording and extracting more data + +Store test results with `recording` data entries which are also scanned by regular expressions +to extract embedded values, for example +[Kernel Latency Benchmarks](https://github.com/zephyrproject-rtos/zephyr/tree/main/tests/benchmarks/latency_measure) +test suite. + +Create the index: +```bash +python3 ./scripts/ci/upload_test_results_es.py --create-index \ + --index zephyr-test-recording-example-2 \ + --map-file zephyr_twister_flat_recording_index.json +``` + +Upload data with 'flattened' test suites creating documents for each `record` data entry +with 3 additional data properties extracted by regular expressions. +```bash +python3 ./scripts/ci/upload_test_results_es.py \ + --index zephyr-test-recording-example-2 \ + --exclude path run_id \ + --flatten recording \ + --transform "{ 'recording_metric': '(?P[^\.]+)\.(?P[^\.]+)\.(?P[^ -]+)' }" \ + ./twister-out/**/twister.json +``` + + +### Memory Footprint results + +To store Memory Footprint data reported in `twister-footprint.json` (see Twister `--footprint-report`). + +Create the index: +```bash +python3 ./scripts/ci/upload_test_results_es.py --create-index \ + --index zephyr-memory-footprint-example \ + --map-file zephyr_twister_flat_footprint_index.json +``` + +Upload data with 'flattened' footprint entries creating documents for each of them. +```bash +python3 ./scripts/ci/upload_test_results_es.py \ + --index zephyr-memory-footprint-example \ + --exclude path run_id \ + --flatten footprint \ + --flatten-list-names "{'children':'name'}" \ + --transform "{ 'footprint_name': '^(?P([^\/]+\/){0,2})(?P([^\/]*\/)*)(?P[^\/]*)$' }" \ + ../footprint_data/**/twister_footprint.json +``` diff --git a/scripts/ci/es_upload/zephyr_twister_flat_footprint_index.json b/scripts/ci/es_upload/zephyr_twister_flat_footprint_index.json new file mode 100644 index 00000000000000..273d4b91aa0537 --- /dev/null +++ b/scripts/ci/es_upload/zephyr_twister_flat_footprint_index.json @@ -0,0 +1,158 @@ +{ + "properties": { + "environment": { + "properties": { + "commit_date": { + "type": "date" + }, + "os": { + "type": "keyword", + "ignore_above": 256 + }, + "run_branch": { + "type": "keyword", + "ignore_above": 256 + }, + "run_workflow": { + "type": "keyword", + "ignore_above": 256 + }, + "run_id": { + "type": "keyword", + "ignore_above": 256 + }, + "run_attempt": { + "type": "integer" + }, + "run_date": { + "type": "date" + }, + "toolchain": { + "type": "keyword", + "ignore_above": 256 + }, + "zephyr_version": { + "type": "keyword", + "ignore_above": 256 + }, + "options": { + "type": "object", + "enabled": true, + "dynamic": true, + "subobjects": true + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "component": { + "type": "keyword", + "ignore_above": 256 + }, + "sub_component": { + "type": "keyword", + "ignore_above": 256 + }, + "arch": { + "type": "keyword", + "ignore_above": 256 + }, + "platform": { + "type": "keyword", + "ignore_above": 256 + }, + "used_ram": { + "type": "unsigned_long" + }, + "used_rom": { + "type": "unsigned_long" + }, + "available_ram": { + "type": "unsigned_long" + }, + "available_rom": { + "type": "unsigned_long" + }, + "retries": { + "type": "integer" + }, + "status": { + "type": "keyword", + "ignore_above": 256 + }, + "build_time": { + "type": "float" + }, + "reason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "log": { + "type": "text" + }, + "footprint_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "footprint_name_depth": { + "type": "integer" + }, + "footprint_identifier": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "footprint_area": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "footprint_path": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "footprint_symbol": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "footprint_address": { + "type": "unsigned_long" + }, + "footprint_size": { + "type": "long" + }, + "footprint_total_size": { + "type": "unsigned_long" + } + } +} diff --git a/scripts/ci/es_upload/zephyr_twister_flat_recording_index.json b/scripts/ci/es_upload/zephyr_twister_flat_recording_index.json new file mode 100644 index 00000000000000..26935ca68cbf08 --- /dev/null +++ b/scripts/ci/es_upload/zephyr_twister_flat_recording_index.json @@ -0,0 +1,195 @@ +{ + "properties": { + "environment": { + "properties": { + "commit_date": { + "type": "date" + }, + "os": { + "type": "keyword", + "ignore_above": 256 + }, + "run_branch": { + "type": "keyword", + "ignore_above": 256 + }, + "run_workflow": { + "type": "keyword", + "ignore_above": 256 + }, + "run_id": { + "type": "keyword", + "ignore_above": 256 + }, + "run_attempt": { + "type": "integer" + }, + "run_date": { + "type": "date" + }, + "toolchain": { + "type": "keyword", + "ignore_above": 256 + }, + "zephyr_version": { + "type": "keyword", + "ignore_above": 256 + }, + "options": { + "type": "object", + "enabled": true, + "dynamic": true, + "subobjects": true + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "component": { + "type": "keyword", + "ignore_above": 256 + }, + "sub_component": { + "type": "keyword", + "ignore_above": 256 + }, + "arch": { + "type": "keyword", + "ignore_above": 256 + }, + "platform": { + "type": "keyword", + "ignore_above": 256 + }, + "path": { + "type": "keyword", + "ignore_above": 256 + }, + "run_id": { + "type": "keyword", + "ignore_above": 256 + }, + "runnable": { + "type": "boolean" + }, + "used_ram": { + "type": "unsigned_long" + }, + "used_rom": { + "type": "unsigned_long" + }, + "available_ram": { + "type": "unsigned_long" + }, + "available_rom": { + "type": "unsigned_long" + }, + "retries": { + "type": "integer" + }, + "dut": { + "type": "keyword", + "ignore_above": 256 + }, + "status": { + "type": "keyword", + "ignore_above": 256 + }, + "execution_time": { + "type": "float" + }, + "build_time": { + "type": "float" + }, + "reason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "log": { + "type": "text" + }, + "testcases": { + "type": "nested", + "properties": { + "execution_time": { + "type": "float" + }, + "identifier": { + "type": "keyword", + "ignore_above": 256 + }, + "reason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording_metric": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording_object": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording_action": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording_details": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording_description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording_cycles": { + "type": "integer" + }, + "recording_nanoseconds": { + "type": "unsigned_long" + } + } +} diff --git a/scripts/ci/es_upload/zephyr_twister_flat_recording_metrics_index.json b/scripts/ci/es_upload/zephyr_twister_flat_recording_metrics_index.json new file mode 100644 index 00000000000000..3527f6f4f7960b --- /dev/null +++ b/scripts/ci/es_upload/zephyr_twister_flat_recording_metrics_index.json @@ -0,0 +1,150 @@ +{ + "properties": { + "environment": { + "properties": { + "commit_date": { + "type": "date" + }, + "os": { + "type": "keyword", + "ignore_above": 256 + }, + "run_branch": { + "type": "keyword", + "ignore_above": 256 + }, + "run_workflow": { + "type": "keyword", + "ignore_above": 256 + }, + "run_id": { + "type": "keyword", + "ignore_above": 256 + }, + "run_attempt": { + "type": "integer" + }, + "run_date": { + "type": "date" + }, + "toolchain": { + "type": "keyword", + "ignore_above": 256 + }, + "zephyr_version": { + "type": "keyword", + "ignore_above": 256 + }, + "options": { + "type": "object", + "enabled": true, + "dynamic": true, + "subobjects": true + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "component": { + "type": "keyword", + "ignore_above": 256 + }, + "sub_component": { + "type": "keyword", + "ignore_above": 256 + }, + "arch": { + "type": "keyword", + "ignore_above": 256 + }, + "platform": { + "type": "keyword", + "ignore_above": 256 + }, + "path": { + "type": "keyword", + "ignore_above": 256 + }, + "run_id": { + "type": "keyword", + "ignore_above": 256 + }, + "runnable": { + "type": "boolean" + }, + "used_ram": { + "type": "unsigned_long" + }, + "used_rom": { + "type": "unsigned_long" + }, + "available_ram": { + "type": "unsigned_long" + }, + "available_rom": { + "type": "unsigned_long" + }, + "retries": { + "type": "integer" + }, + "dut": { + "type": "keyword", + "ignore_above": 256 + }, + "status": { + "type": "keyword", + "ignore_above": 256 + }, + "execution_time": { + "type": "float" + }, + "build_time": { + "type": "float" + }, + "reason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "log": { + "type": "text" + }, + "testcases": { + "type": "nested", + "properties": { + "execution_time": { + "type": "float" + }, + "identifier": { + "type": "keyword", + "ignore_above": 256 + }, + "reason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording_metrics": { + "type": "object", + "enabled": true, + "dynamic": true, + "subobjects": true + } + } +} diff --git a/scripts/ci/es_upload/zephyr_twister_index.json b/scripts/ci/es_upload/zephyr_twister_index.json new file mode 100644 index 00000000000000..4c94a3f8cbba3f --- /dev/null +++ b/scripts/ci/es_upload/zephyr_twister_index.json @@ -0,0 +1,149 @@ +{ + "properties": { + "environment": { + "properties": { + "commit_date": { + "type": "date" + }, + "os": { + "type": "keyword", + "ignore_above": 256 + }, + "run_branch": { + "type": "keyword", + "ignore_above": 256 + }, + "run_workflow": { + "type": "keyword", + "ignore_above": 256 + }, + "run_id": { + "type": "keyword", + "ignore_above": 256 + }, + "run_attempt": { + "type": "integer" + }, + "run_date": { + "type": "date" + }, + "toolchain": { + "type": "keyword", + "ignore_above": 256 + }, + "zephyr_version": { + "type": "keyword", + "ignore_above": 256 + }, + "options": { + "type": "object", + "enabled": true, + "dynamic": true, + "subobjects": true + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "component": { + "type": "keyword", + "ignore_above": 256 + }, + "sub_component": { + "type": "keyword", + "ignore_above": 256 + }, + "arch": { + "type": "keyword", + "ignore_above": 256 + }, + "platform": { + "type": "keyword", + "ignore_above": 256 + }, + "path": { + "type": "keyword", + "ignore_above": 256 + }, + "run_id": { + "type": "keyword", + "ignore_above": 256 + }, + "runnable": { + "type": "boolean" + }, + "used_ram": { + "type": "unsigned_long" + }, + "used_rom": { + "type": "unsigned_long" + }, + "available_ram": { + "type": "unsigned_long" + }, + "available_rom": { + "type": "unsigned_long" + }, + "retries": { + "type": "integer" + }, + "dut": { + "type": "keyword", + "ignore_above": 256 + }, + "status": { + "type": "keyword", + "ignore_above": 256 + }, + "execution_time": { + "type": "float" + }, + "build_time": { + "type": "float" + }, + "reason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "log": { + "type": "text" + }, + "testcases": { + "properties": { + "execution_time": { + "type": "float" + }, + "identifier": { + "type": "keyword", + "ignore_above": 256 + }, + "reason": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recording": { + "type": "object", + "enabled": true, + "dynamic": true, + "subobjects": true + } + } +}