From cab940ce56fcfa78c30a36a46709a5f1f475e919 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 13:23:19 +0200 Subject: [PATCH 01/22] otel: rename LogRecord functions to snake_case Signed-off-by: Attila Szakacs --- .../grpc/otel/filterx/object-otel-logrecord.cpp | 14 +++++++------- .../grpc/otel/filterx/object-otel-logrecord.hpp | 8 ++++---- modules/grpc/otel/tests/test-otel-filterx.cpp | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index c189007529..320e764695 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -65,14 +65,14 @@ LogRecord::LogRecord(const LogRecord &o, FilterXOtelLogRecord *super_) : super(s } std::string -LogRecord::Marshal(void) +LogRecord::marshal(void) { std::string serializedString = this->logRecord.SerializePartialAsString(); return serializedString; } bool -LogRecord::SetField(const gchar *attribute, FilterXObject **value) +LogRecord::set_field(const gchar *attribute, FilterXObject **value) { try { @@ -96,7 +96,7 @@ LogRecord::SetField(const gchar *attribute, FilterXObject **value) } FilterXObject * -LogRecord::GetField(const gchar *attribute) +LogRecord::get_field(const gchar *attribute) { try { @@ -113,7 +113,7 @@ LogRecord::GetField(const gchar *attribute) } const opentelemetry::proto::logs::v1::LogRecord & -LogRecord::GetValue() const +LogRecord::get_value() const { return this->logRecord; } @@ -160,7 +160,7 @@ _setattr(FilterXObject *s, FilterXObject *attr, FilterXObject **new_value) { FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; - return self->cpp->SetField(filterx_string_get_value(attr, NULL), new_value); + return self->cpp->set_field(filterx_string_get_value(attr, NULL), new_value); } static FilterXObject * @@ -168,7 +168,7 @@ _getattr(FilterXObject *s, FilterXObject *attr) { FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; - return self->cpp->GetField(filterx_string_get_value(attr, NULL)); + return self->cpp->get_field(filterx_string_get_value(attr, NULL)); } static gboolean @@ -182,7 +182,7 @@ _marshal(FilterXObject *s, GString *repr, LogMessageValueType *t) { FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; - std::string serialized = self->cpp->Marshal(); + std::string serialized = self->cpp->marshal(); g_string_truncate(repr, 0); g_string_append_len(repr, serialized.c_str(), serialized.length()); diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.hpp b/modules/grpc/otel/filterx/object-otel-logrecord.hpp index 0fb5bd54ea..accaa6fdd9 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.hpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.hpp @@ -48,10 +48,10 @@ class LogRecord LogRecord(FilterXOtelLogRecord *super, FilterXObject *protobuf_object); LogRecord(LogRecord &o) = delete; LogRecord(LogRecord &&o) = delete; - bool SetField(const gchar *attribute, FilterXObject **value); - std::string Marshal(void); - FilterXObject *GetField(const gchar *attribute); - const opentelemetry::proto::logs::v1::LogRecord &GetValue() const; + bool set_field(const gchar *attribute, FilterXObject **value); + std::string marshal(void); + FilterXObject *get_field(const gchar *attribute); + const opentelemetry::proto::logs::v1::LogRecord &get_value() const; private: FilterXOtelLogRecord *super; opentelemetry::proto::logs::v1::LogRecord logRecord; diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 23a94937b5..9f91a44ebc 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -155,7 +155,7 @@ Test(otel_filterx, logrecord_empty) FilterXOtelLogRecord *filterx_otel_logrecord = (FilterXOtelLogRecord *) filterx_otel_logrecord_new_from_args(NULL); cr_assert(filterx_otel_logrecord); - cr_assert(MessageDifferencer::Equals(LogRecord(), filterx_otel_logrecord->cpp->GetValue())); + cr_assert(MessageDifferencer::Equals(LogRecord(), filterx_otel_logrecord->cpp->get_value())); filterx_object_unref(&filterx_otel_logrecord->super); } @@ -176,7 +176,7 @@ Test(otel_filterx, logrecord_from_protobuf) FilterXOtelLogRecord *filterx_otel_logrecord = (FilterXOtelLogRecord *) filterx_otel_logrecord_new_from_args(args); cr_assert(filterx_otel_logrecord); - const LogRecord &log_record_from_filterx = filterx_otel_logrecord->cpp->GetValue(); + const LogRecord &log_record_from_filterx = filterx_otel_logrecord->cpp->get_value(); cr_assert(MessageDifferencer::Equals(log_record, log_record_from_filterx)); filterx_object_unref(&filterx_otel_logrecord->super); @@ -726,7 +726,7 @@ Test(otel_filterx, kvlist_through_logrecord) KeyValue *expected_logrecord_inner_attr_2 = expected_logrecord_inner_kvlist->add_values(); expected_logrecord_inner_attr_2->set_key("key_2"); expected_logrecord_inner_attr_2->mutable_value()->set_string_value("baz"); - cr_assert(MessageDifferencer::Equals(expected_logrecord, ((FilterXOtelLogRecord *) fx_logrecord)->cpp->GetValue())); + cr_assert(MessageDifferencer::Equals(expected_logrecord, ((FilterXOtelLogRecord *) fx_logrecord)->cpp->get_value())); KeyValueList expected_kvlist; KeyValue *expected_kvlist_value_0 = expected_kvlist.add_values(); @@ -973,7 +973,7 @@ Test(otel_filterx, array_through_logrecord) expected_logrecord_array->add_values()->set_string_value("bar"); ArrayValue *inner_array = expected_logrecord_array->add_values()->mutable_array_value(); inner_array->add_values()->set_string_value("baz"); - cr_assert(MessageDifferencer::Equals(expected_logrecord, ((FilterXOtelLogRecord *) fx_logrecord)->cpp->GetValue())); + cr_assert(MessageDifferencer::Equals(expected_logrecord, ((FilterXOtelLogRecord *) fx_logrecord)->cpp->get_value())); ArrayValue expected_array; expected_array.add_values()->set_string_value("foo"); From b1de55d8f9b005db08f50b49dd630f949aad5771 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 13:32:24 +0200 Subject: [PATCH 02/22] otel: pass const std::string & where possible This can save a copy. Signed-off-by: Attila Szakacs --- modules/grpc/otel/filterx/protobuf-field.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/grpc/otel/filterx/protobuf-field.hpp b/modules/grpc/otel/filterx/protobuf-field.hpp index 7bc8c5716e..1a174bb9c0 100644 --- a/modules/grpc/otel/filterx/protobuf-field.hpp +++ b/modules/grpc/otel/filterx/protobuf-field.hpp @@ -43,7 +43,7 @@ struct ProtoReflectors const google::protobuf::Descriptor *descriptor; const google::protobuf::FieldDescriptor *fieldDescriptor; google::protobuf::FieldDescriptor::Type fieldType; - ProtoReflectors(const google::protobuf::Message &message, std::string fieldName) + ProtoReflectors(const google::protobuf::Message &message, const std::string &fieldName) { this->reflection = message.GetReflection(); this->descriptor = message.GetDescriptor(); @@ -71,7 +71,7 @@ struct ProtoReflectors class ProtobufField { public: - FilterXObject *Get(google::protobuf::Message *message, std::string fieldName) + FilterXObject *Get(google::protobuf::Message *message, const std::string &fieldName) { try { @@ -84,7 +84,8 @@ class ProtobufField return nullptr; } }; - bool Set(google::protobuf::Message *message, std::string fieldName, FilterXObject *object, FilterXObject **assoc_object) + bool Set(google::protobuf::Message *message, const std::string &fieldName, FilterXObject *object, + FilterXObject **assoc_object) { try { From 41ed14d844fd91bd9157e5a8c36171c960ed769c Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 13:46:04 +0200 Subject: [PATCH 03/22] otel: add extract_string_from_object() helper fn Signed-off-by: Attila Szakacs --- modules/grpc/otel/filterx/protobuf-field.cpp | 16 ++++++++++++++++ modules/grpc/otel/filterx/protobuf-field.hpp | 2 ++ 2 files changed, 18 insertions(+) diff --git a/modules/grpc/otel/filterx/protobuf-field.cpp b/modules/grpc/otel/filterx/protobuf-field.cpp index e6bbcd54d9..744993cc45 100644 --- a/modules/grpc/otel/filterx/protobuf-field.cpp +++ b/modules/grpc/otel/filterx/protobuf-field.cpp @@ -25,6 +25,7 @@ #include "compat/cpp-start.h" #include "filterx/object-string.h" +#include "filterx/object-message-value.h" #include "filterx/object-datetime.h" #include "filterx/object-primitive.h" #include "scratch-buffers.h" @@ -365,3 +366,18 @@ ProtobufField *syslogng::grpc::otel::protobuf_converter_by_type(google::protobuf g_assert(fieldType <= google::protobuf::FieldDescriptor::MAX_TYPE && fieldType > 0); return all_protobuf_converters()[fieldType - 1].get(); } + +std::string +syslogng::grpc::otel::extract_string_from_object(FilterXObject *object) +{ + gsize len; + const gchar *key_c_str = filterx_string_get_value(object, &len); + + if (!key_c_str) + key_c_str = filterx_message_value_get_value(object, &len); + + if (!key_c_str) + throw std::runtime_error("not a string instance"); + + return std::string{key_c_str, len}; +} diff --git a/modules/grpc/otel/filterx/protobuf-field.hpp b/modules/grpc/otel/filterx/protobuf-field.hpp index 1a174bb9c0..5025b1f57f 100644 --- a/modules/grpc/otel/filterx/protobuf-field.hpp +++ b/modules/grpc/otel/filterx/protobuf-field.hpp @@ -115,6 +115,8 @@ class ProtobufField std::unique_ptr *all_protobuf_converters(); ProtobufField *protobuf_converter_by_type(google::protobuf::FieldDescriptor::Type fieldType); +std::string extract_string_from_object(FilterXObject *object); + } } } From bab170d86e50b3082261137f82ac844121612c49 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 13:47:01 +0200 Subject: [PATCH 04/22] otel: port LogRecord to implement the Dict interface Signed-off-by: Attila Szakacs --- .../otel/filterx/object-otel-logrecord.cpp | 90 ++++++++++--------- .../otel/filterx/object-otel-logrecord.hpp | 8 +- modules/grpc/otel/tests/test-otel-filterx.cpp | 4 +- 3 files changed, 54 insertions(+), 48 deletions(-) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 320e764695..d9d7ca8976 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -72,14 +72,16 @@ LogRecord::marshal(void) } bool -LogRecord::set_field(const gchar *attribute, FilterXObject **value) +LogRecord::set_subscript(FilterXObject *key, FilterXObject **value) { try { - ProtoReflectors reflectors(this->logRecord, attribute); + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(this->logRecord, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + FilterXObject *assoc_object = NULL; - if (!otel_converter_by_field_descriptor(reflectors.fieldDescriptor)->Set(&this->logRecord, attribute, *value, - &assoc_object)) + if (!converter->Set(&this->logRecord, key_str, *value, &assoc_object)) return false; filterx_object_unref(*value); @@ -88,26 +90,23 @@ LogRecord::set_field(const gchar *attribute, FilterXObject **value) } catch(const std::exception &ex) { - msg_error("protobuf-field: Failed to set field:", - evt_tag_str("message", ex.what()), - evt_tag_str("field_name", attribute)); return false; } } FilterXObject * -LogRecord::get_field(const gchar *attribute) +LogRecord::get_subscript(FilterXObject *key) { try { - ProtoReflectors reflectors(this->logRecord, attribute); - return otel_converter_by_field_descriptor(reflectors.fieldDescriptor)->Get(&this->logRecord, attribute); + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(this->logRecord, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->Get(&this->logRecord, key_str); } catch(const std::exception &ex) { - msg_error("protobuf-field: Failed to get field:", - evt_tag_str("message", ex.what()), - evt_tag_str("field_name", attribute)); return nullptr; } } @@ -126,26 +125,6 @@ grpc_otel_filterx_logrecord_contruct_new(Plugin *self) return (gpointer) &filterx_otel_logrecord_new_from_args; } -FilterXObject * -_filterx_otel_logrecord_clone(FilterXObject *s) -{ - FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; - - FilterXOtelLogRecord *clone = g_new0(FilterXOtelLogRecord, 1); - filterx_object_init_instance((FilterXObject *) clone, &FILTERX_TYPE_NAME(otel_logrecord)); - - try - { - clone->cpp = new LogRecord(*self->cpp, clone); - } - catch (const std::runtime_error &) - { - g_assert_not_reached(); - } - - return &clone->super; -} - static void _free(FilterXObject *s) { @@ -156,19 +135,19 @@ _free(FilterXObject *s) } static gboolean -_setattr(FilterXObject *s, FilterXObject *attr, FilterXObject **new_value) +_set_subscript(FilterXDict *s, FilterXObject *key, FilterXObject **new_value) { FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; - return self->cpp->set_field(filterx_string_get_value(attr, NULL), new_value); + return self->cpp->set_subscript(key, new_value); } static FilterXObject * -_getattr(FilterXObject *s, FilterXObject *attr) +_get_subscript(FilterXDict *s, FilterXObject *key) { FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; - return self->cpp->get_field(filterx_string_get_value(attr, NULL)); + return self->cpp->get_subscript(key); } static gboolean @@ -190,11 +169,40 @@ _marshal(FilterXObject *s, GString *repr, LogMessageValueType *t) return TRUE; } +static void +_init_instance(FilterXOtelLogRecord *self) +{ + filterx_dict_init_instance(&self->super, &FILTERX_TYPE_NAME(otel_logrecord)); + + self->super.get_subscript = _get_subscript; + self->super.set_subscript = _set_subscript; +} + +FilterXObject * +_filterx_otel_logrecord_clone(FilterXObject *s) +{ + FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; + + FilterXOtelLogRecord *clone = g_new0(FilterXOtelLogRecord, 1); + _init_instance(clone); + + try + { + clone->cpp = new LogRecord(*self->cpp, clone); + } + catch (const std::runtime_error &) + { + g_assert_not_reached(); + } + + return &clone->super.super; +} + FilterXObject * filterx_otel_logrecord_new_from_args(GPtrArray *args) { FilterXOtelLogRecord *self = g_new0(FilterXOtelLogRecord, 1); - filterx_object_init_instance((FilterXObject *) self, &FILTERX_TYPE_NAME(otel_logrecord)); + _init_instance(self); try { @@ -208,11 +216,11 @@ filterx_otel_logrecord_new_from_args(GPtrArray *args) catch (const std::runtime_error &e) { msg_error("FilterX: Failed to create OTel LogRecord object", evt_tag_str("error", e.what())); - filterx_object_unref(&self->super); + filterx_object_unref(&self->super.super); return NULL; } - return &self->super; + return &self->super.super; } FILTERX_DEFINE_TYPE(otel_logrecord, FILTERX_TYPE_NAME(object), @@ -220,7 +228,5 @@ FILTERX_DEFINE_TYPE(otel_logrecord, FILTERX_TYPE_NAME(object), .marshal = _marshal, .clone = _filterx_otel_logrecord_clone, .truthy = _truthy, - .getattr = _getattr, - .setattr = _setattr, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.hpp b/modules/grpc/otel/filterx/object-otel-logrecord.hpp index accaa6fdd9..e54f9a86ce 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.hpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.hpp @@ -26,7 +26,7 @@ #include "syslog-ng.h" #include "compat/cpp-start.h" -#include "filterx/filterx-object.h" +#include "filterx/object-dict-interface.h" #include "object-otel.h" #include "compat/cpp-end.h" @@ -48,9 +48,9 @@ class LogRecord LogRecord(FilterXOtelLogRecord *super, FilterXObject *protobuf_object); LogRecord(LogRecord &o) = delete; LogRecord(LogRecord &&o) = delete; - bool set_field(const gchar *attribute, FilterXObject **value); std::string marshal(void); - FilterXObject *get_field(const gchar *attribute); + FilterXObject *get_subscript(FilterXObject *key); + bool set_subscript(FilterXObject *key, FilterXObject **value); const opentelemetry::proto::logs::v1::LogRecord &get_value() const; private: FilterXOtelLogRecord *super; @@ -66,7 +66,7 @@ class LogRecord struct FilterXOtelLogRecord_ { - FilterXObject super; + FilterXDict super; syslogng::grpc::otel::filterx::LogRecord *cpp; }; diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 9f91a44ebc..aea8a6faae 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -157,7 +157,7 @@ Test(otel_filterx, logrecord_empty) cr_assert(MessageDifferencer::Equals(LogRecord(), filterx_otel_logrecord->cpp->get_value())); - filterx_object_unref(&filterx_otel_logrecord->super); + filterx_object_unref(&filterx_otel_logrecord->super.super); } Test(otel_filterx, logrecord_from_protobuf) @@ -179,7 +179,7 @@ Test(otel_filterx, logrecord_from_protobuf) const LogRecord &log_record_from_filterx = filterx_otel_logrecord->cpp->get_value(); cr_assert(MessageDifferencer::Equals(log_record, log_record_from_filterx)); - filterx_object_unref(&filterx_otel_logrecord->super); + filterx_object_unref(&filterx_otel_logrecord->super.super); g_ptr_array_free(args, TRUE); } From 34614445095d1f3e9a50b396c20c93e997b56f0e Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 14:20:52 +0200 Subject: [PATCH 05/22] otel: add get_protobuf_message_set_field_count() Signed-off-by: Attila Szakacs --- modules/grpc/otel/filterx/protobuf-field.cpp | 9 +++++++++ modules/grpc/otel/filterx/protobuf-field.hpp | 2 ++ 2 files changed, 11 insertions(+) diff --git a/modules/grpc/otel/filterx/protobuf-field.cpp b/modules/grpc/otel/filterx/protobuf-field.cpp index 744993cc45..69d52af910 100644 --- a/modules/grpc/otel/filterx/protobuf-field.cpp +++ b/modules/grpc/otel/filterx/protobuf-field.cpp @@ -381,3 +381,12 @@ syslogng::grpc::otel::extract_string_from_object(FilterXObject *object) return std::string{key_c_str, len}; } + +uint64_t +syslogng::grpc::otel::get_protobuf_message_set_field_count(const google::protobuf::Message &message) +{ + const google::protobuf::Reflection *reflection = message.GetReflection(); + std::vector fields; + reflection->ListFields(message, &fields); + return fields.size(); +} diff --git a/modules/grpc/otel/filterx/protobuf-field.hpp b/modules/grpc/otel/filterx/protobuf-field.hpp index 5025b1f57f..883f4c6f72 100644 --- a/modules/grpc/otel/filterx/protobuf-field.hpp +++ b/modules/grpc/otel/filterx/protobuf-field.hpp @@ -117,6 +117,8 @@ ProtobufField *protobuf_converter_by_type(google::protobuf::FieldDescriptor::Typ std::string extract_string_from_object(FilterXObject *object); +uint64_t get_protobuf_message_set_field_count(const google::protobuf::Message &message); + } } } From 17f9228883160c2a824c4fecbd8585deb48f0653 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 16:32:44 +0200 Subject: [PATCH 06/22] otel: implement len() for LogRecord Signed-off-by: Attila Szakacs --- .../otel/filterx/object-otel-logrecord.cpp | 18 ++++++++++++- .../otel/filterx/object-otel-logrecord.hpp | 1 + modules/grpc/otel/tests/test-otel-filterx.cpp | 26 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index d9d7ca8976..6116c82327 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -38,6 +38,7 @@ #include #include #include +#include using namespace syslogng::grpc::otel::filterx; @@ -111,6 +112,12 @@ LogRecord::get_subscript(FilterXObject *key) } } +uint64_t +LogRecord::len() const +{ + return get_protobuf_message_set_field_count(logRecord); +} + const opentelemetry::proto::logs::v1::LogRecord & LogRecord::get_value() const { @@ -150,6 +157,14 @@ _get_subscript(FilterXDict *s, FilterXObject *key) return self->cpp->get_subscript(key); } +static guint64 +_len(FilterXDict *s) +{ + FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; + + return self->cpp->len(); +} + static gboolean _truthy(FilterXObject *s) { @@ -176,6 +191,7 @@ _init_instance(FilterXOtelLogRecord *self) self->super.get_subscript = _get_subscript; self->super.set_subscript = _set_subscript; + self->super.len = _len; } FilterXObject * @@ -223,7 +239,7 @@ filterx_otel_logrecord_new_from_args(GPtrArray *args) return &self->super.super; } -FILTERX_DEFINE_TYPE(otel_logrecord, FILTERX_TYPE_NAME(object), +FILTERX_DEFINE_TYPE(otel_logrecord, FILTERX_TYPE_NAME(dict), .is_mutable = TRUE, .marshal = _marshal, .clone = _filterx_otel_logrecord_clone, diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.hpp b/modules/grpc/otel/filterx/object-otel-logrecord.hpp index e54f9a86ce..eb70855a93 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.hpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.hpp @@ -51,6 +51,7 @@ class LogRecord std::string marshal(void); FilterXObject *get_subscript(FilterXObject *key); bool set_subscript(FilterXObject *key, FilterXObject **value); + uint64_t len() const; const opentelemetry::proto::logs::v1::LogRecord &get_value() const; private: FilterXOtelLogRecord *super; diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index aea8a6faae..7f8c7810ff 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -217,6 +217,32 @@ Test(otel_filterx, logrecord_too_many_args) g_ptr_array_free(args, TRUE); } +Test(otel_filterx, logrecord_len) +{ + FilterXObject *logrecord = filterx_otel_logrecord_new_from_args(NULL); + FilterXObject *body = filterx_string_new("body", -1); + FilterXObject *body_val = filterx_string_new("body_val", -1); + FilterXObject *time_unix_nano = filterx_string_new("time_unix_nano", -1); + FilterXObject *time_unix_nano_val = filterx_integer_new(123); + + guint64 len; + cr_assert(filterx_object_len(logrecord, &len)); + cr_assert_eq(len, 0); + + cr_assert(filterx_object_set_subscript(logrecord, body, &body_val)); + cr_assert(filterx_object_len(logrecord, &len)); + cr_assert_eq(len, 1); + + cr_assert(filterx_object_set_subscript(logrecord, time_unix_nano, &time_unix_nano_val)); + cr_assert(filterx_object_len(logrecord, &len)); + cr_assert_eq(len, 2); + + filterx_object_unref(time_unix_nano); + filterx_object_unref(time_unix_nano_val); + filterx_object_unref(body); + filterx_object_unref(body_val); + filterx_object_unref(logrecord); +} /* Resource */ From 3646630f38b6c2d4fe149c010e7589626252a230 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 15:07:03 +0200 Subject: [PATCH 07/22] otel: add Unset() to ProtobufField Signed-off-by: Attila Szakacs --- modules/grpc/otel/filterx/protobuf-field.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/modules/grpc/otel/filterx/protobuf-field.hpp b/modules/grpc/otel/filterx/protobuf-field.hpp index 883f4c6f72..0d46baffdd 100644 --- a/modules/grpc/otel/filterx/protobuf-field.hpp +++ b/modules/grpc/otel/filterx/protobuf-field.hpp @@ -104,6 +104,20 @@ class ProtobufField return false; } } + bool Unset(google::protobuf::Message *message, const std::string &fieldName) + { + try + { + ProtoReflectors reflectors(*message, fieldName); + reflectors.reflection->ClearField(message, reflectors.fieldDescriptor); + return true; + } + catch(const std::exception &ex) + { + msg_error("protobuf-field: Failed to unset field:", evt_tag_str("message", ex.what())); + return false; + } + } virtual ~ProtobufField() {} protected: From 9ce814a5568ad87b5b3a7b6852040853b2fdd7f5 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 15:07:21 +0200 Subject: [PATCH 08/22] otel: implement unset for LogRecord Signed-off-by: Attila Szakacs --- .../otel/filterx/object-otel-logrecord.cpp | 26 +++++++++++++++++++ .../otel/filterx/object-otel-logrecord.hpp | 1 + modules/grpc/otel/tests/test-otel-filterx.cpp | 10 ++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 6116c82327..13885b7187 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -112,6 +112,23 @@ LogRecord::get_subscript(FilterXObject *key) } } +bool +LogRecord::unset_key(FilterXObject *key) +{ + try + { + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(this->logRecord, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->Unset(&this->logRecord, key_str); + } + catch(const std::exception &ex) + { + return false; + } +} + uint64_t LogRecord::len() const { @@ -157,6 +174,14 @@ _get_subscript(FilterXDict *s, FilterXObject *key) return self->cpp->get_subscript(key); } +static gboolean +_unset_key(FilterXDict *s, FilterXObject *key) +{ + FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; + + return self->cpp->unset_key(key); +} + static guint64 _len(FilterXDict *s) { @@ -191,6 +216,7 @@ _init_instance(FilterXOtelLogRecord *self) self->super.get_subscript = _get_subscript; self->super.set_subscript = _set_subscript; + self->super.unset_key = _unset_key; self->super.len = _len; } diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.hpp b/modules/grpc/otel/filterx/object-otel-logrecord.hpp index eb70855a93..ffddef753f 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.hpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.hpp @@ -51,6 +51,7 @@ class LogRecord std::string marshal(void); FilterXObject *get_subscript(FilterXObject *key); bool set_subscript(FilterXObject *key, FilterXObject **value); + bool unset_key(FilterXObject *key); uint64_t len() const; const opentelemetry::proto::logs::v1::LogRecord &get_value() const; private: diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 7f8c7810ff..0db772d4b5 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -217,7 +217,7 @@ Test(otel_filterx, logrecord_too_many_args) g_ptr_array_free(args, TRUE); } -Test(otel_filterx, logrecord_len) +Test(otel_filterx, logrecord_len_and_unset) { FilterXObject *logrecord = filterx_otel_logrecord_new_from_args(NULL); FilterXObject *body = filterx_string_new("body", -1); @@ -237,6 +237,14 @@ Test(otel_filterx, logrecord_len) cr_assert(filterx_object_len(logrecord, &len)); cr_assert_eq(len, 2); + cr_assert(filterx_object_unset_key(logrecord, body)); + cr_assert(filterx_object_len(logrecord, &len)); + cr_assert_eq(len, 1); + + cr_assert(filterx_object_unset_key(logrecord, time_unix_nano)); + cr_assert(filterx_object_len(logrecord, &len)); + cr_assert_eq(len, 0); + filterx_object_unref(time_unix_nano); filterx_object_unref(time_unix_nano_val); filterx_object_unref(body); From 6442d2fc50e2f1c33a167cfee34f6112097d7216 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 15:16:15 +0200 Subject: [PATCH 09/22] otel: add IsSet() to ProtobufField Signed-off-by: Attila Szakacs --- modules/grpc/otel/filterx/protobuf-field.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/grpc/otel/filterx/protobuf-field.hpp b/modules/grpc/otel/filterx/protobuf-field.hpp index 0d46baffdd..ba181b9bba 100644 --- a/modules/grpc/otel/filterx/protobuf-field.hpp +++ b/modules/grpc/otel/filterx/protobuf-field.hpp @@ -118,6 +118,19 @@ class ProtobufField return false; } } + bool IsSet(google::protobuf::Message *message, const std::string &fieldName) + { + try + { + ProtoReflectors reflectors(*message, fieldName); + return reflectors.reflection->HasField(*message, reflectors.fieldDescriptor); + } + catch(const std::exception &ex) + { + msg_error("protobuf-field: Failed to check field:", evt_tag_str("message", ex.what())); + return false; + } + } virtual ~ProtobufField() {} protected: From 2ed39cd1cd046bdb5e38ba137146a5c9db8f4d57 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 15:16:27 +0200 Subject: [PATCH 10/22] otel: implement is_key_set for LogRecord Signed-off-by: Attila Szakacs --- .../otel/filterx/object-otel-logrecord.cpp | 26 +++++++++++++++++++ .../otel/filterx/object-otel-logrecord.hpp | 1 + modules/grpc/otel/tests/test-otel-filterx.cpp | 8 +++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 13885b7187..468d9ad4c5 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -129,6 +129,23 @@ LogRecord::unset_key(FilterXObject *key) } } +bool +LogRecord::is_key_set(FilterXObject *key) +{ + try + { + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(this->logRecord, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->IsSet(&this->logRecord, key_str); + } + catch(const std::exception &ex) + { + return false; + } +} + uint64_t LogRecord::len() const { @@ -182,6 +199,14 @@ _unset_key(FilterXDict *s, FilterXObject *key) return self->cpp->unset_key(key); } +static gboolean +_is_key_set(FilterXDict *s, FilterXObject *key) +{ + FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; + + return self->cpp->is_key_set(key); +} + static guint64 _len(FilterXDict *s) { @@ -217,6 +242,7 @@ _init_instance(FilterXOtelLogRecord *self) self->super.get_subscript = _get_subscript; self->super.set_subscript = _set_subscript; self->super.unset_key = _unset_key; + self->super.is_key_set = _is_key_set; self->super.len = _len; } diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.hpp b/modules/grpc/otel/filterx/object-otel-logrecord.hpp index ffddef753f..48e92e46c7 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.hpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.hpp @@ -52,6 +52,7 @@ class LogRecord FilterXObject *get_subscript(FilterXObject *key); bool set_subscript(FilterXObject *key, FilterXObject **value); bool unset_key(FilterXObject *key); + bool is_key_set(FilterXObject *key); uint64_t len() const; const opentelemetry::proto::logs::v1::LogRecord &get_value() const; private: diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 0db772d4b5..e744161c79 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -217,7 +217,7 @@ Test(otel_filterx, logrecord_too_many_args) g_ptr_array_free(args, TRUE); } -Test(otel_filterx, logrecord_len_and_unset) +Test(otel_filterx, logrecord_len_and_unset_and_is_key_set) { FilterXObject *logrecord = filterx_otel_logrecord_new_from_args(NULL); FilterXObject *body = filterx_string_new("body", -1); @@ -229,21 +229,27 @@ Test(otel_filterx, logrecord_len_and_unset) cr_assert(filterx_object_len(logrecord, &len)); cr_assert_eq(len, 0); + cr_assert_not(filterx_object_is_key_set(logrecord, body)); cr_assert(filterx_object_set_subscript(logrecord, body, &body_val)); cr_assert(filterx_object_len(logrecord, &len)); cr_assert_eq(len, 1); + cr_assert(filterx_object_is_key_set(logrecord, body)); + cr_assert_not(filterx_object_is_key_set(logrecord, time_unix_nano)); cr_assert(filterx_object_set_subscript(logrecord, time_unix_nano, &time_unix_nano_val)); cr_assert(filterx_object_len(logrecord, &len)); cr_assert_eq(len, 2); + cr_assert(filterx_object_is_key_set(logrecord, time_unix_nano)); cr_assert(filterx_object_unset_key(logrecord, body)); cr_assert(filterx_object_len(logrecord, &len)); cr_assert_eq(len, 1); + cr_assert_not(filterx_object_is_key_set(logrecord, body)); cr_assert(filterx_object_unset_key(logrecord, time_unix_nano)); cr_assert(filterx_object_len(logrecord, &len)); cr_assert_eq(len, 0); + cr_assert_not(filterx_object_is_key_set(logrecord, time_unix_nano)); filterx_object_unref(time_unix_nano); filterx_object_unref(time_unix_nano_val); From 7527ce6dfa97aeb8dd0cca4d9156a899e9088302 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 16:07:16 +0200 Subject: [PATCH 11/22] otel: add iter_on_otel_protobuf_message_fields() Signed-off-by: Attila Szakacs --- modules/grpc/otel/filterx/otel-field.cpp | 41 ++++++++++++++++++++++++ modules/grpc/otel/filterx/otel-field.hpp | 7 ++++ 2 files changed, 48 insertions(+) diff --git a/modules/grpc/otel/filterx/otel-field.cpp b/modules/grpc/otel/filterx/otel-field.cpp index 7130375a7c..6d09b2c498 100644 --- a/modules/grpc/otel/filterx/otel-field.cpp +++ b/modules/grpc/otel/filterx/otel-field.cpp @@ -333,3 +333,44 @@ ProtobufField *syslogng::grpc::otel::otel_converter_by_field_descriptor(const Fi const FieldDescriptor::Type fieldType = fd->type(); return otel_converter_by_type(fieldType); } + +bool +syslogng::grpc::otel::iter_on_otel_protobuf_message_fields(google::protobuf::Message &message, FilterXDictIterFunc func, + void *user_data) +{ + const google::protobuf::Reflection *reflection = message.GetReflection(); + std::vector fields; + reflection->ListFields(message, &fields); + + try + { + for (const google::protobuf::FieldDescriptor *field : fields) + { + const std::string &name = field->name(); + ProtoReflectors field_reflectors(message, name); + ProtobufField *converter = syslogng::grpc::otel::otel_converter_by_field_descriptor(field_reflectors.fieldDescriptor); + + FilterXObject *key = filterx_string_new(name.c_str(), name.size()); + FilterXObject *value = converter->Get(&message, name); + if (!value) + { + filterx_object_unref(key); + return false; + } + + bool success = func(key, value, user_data); + + filterx_object_unref(key); + filterx_object_unref(value); + + if (!success) + return false; + } + } + catch (const std::exception &) + { + return false; + } + + return true; +} diff --git a/modules/grpc/otel/filterx/otel-field.hpp b/modules/grpc/otel/filterx/otel-field.hpp index 8231b8758b..66da5114ad 100644 --- a/modules/grpc/otel/filterx/otel-field.hpp +++ b/modules/grpc/otel/filterx/otel-field.hpp @@ -26,6 +26,10 @@ #include "syslog-ng.h" #include "protobuf-field.hpp" +#include "compat/cpp-start.h" +#include "filterx/object-dict-interface.h" +#include "compat/cpp-end.h" + namespace syslogng { namespace grpc { namespace otel { @@ -50,6 +54,9 @@ extern AnyField any_field_converter; ProtobufField *otel_converter_by_type(FieldDescriptor::Type fieldType); ProtobufField *otel_converter_by_field_descriptor(const FieldDescriptor *fd); +bool iter_on_otel_protobuf_message_fields(google::protobuf::Message &message, FilterXDictIterFunc func, + void *user_data); + } } } From 45848ae8cccaf98efe861739563518c9322dfa9f Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 16:33:35 +0200 Subject: [PATCH 12/22] otel: implement iter() for LogRecord Signed-off-by: Attila Szakacs --- .../otel/filterx/object-otel-logrecord.cpp | 15 +++++++ .../otel/filterx/object-otel-logrecord.hpp | 1 + modules/grpc/otel/tests/test-otel-filterx.cpp | 39 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 468d9ad4c5..720d7cfa0f 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -152,6 +152,12 @@ LogRecord::len() const return get_protobuf_message_set_field_count(logRecord); } +bool +LogRecord::iter(FilterXDictIterFunc func, void *user_data) +{ + return iter_on_otel_protobuf_message_fields(logRecord, func, user_data); +} + const opentelemetry::proto::logs::v1::LogRecord & LogRecord::get_value() const { @@ -215,6 +221,14 @@ _len(FilterXDict *s) return self->cpp->len(); } +static gboolean +_iter(FilterXDict *s, FilterXDictIterFunc func, gpointer user_data) +{ + FilterXOtelLogRecord *self = (FilterXOtelLogRecord *) s; + + return self->cpp->iter(func, user_data); +} + static gboolean _truthy(FilterXObject *s) { @@ -244,6 +258,7 @@ _init_instance(FilterXOtelLogRecord *self) self->super.unset_key = _unset_key; self->super.is_key_set = _is_key_set; self->super.len = _len; + self->super.iter = _iter; } FilterXObject * diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.hpp b/modules/grpc/otel/filterx/object-otel-logrecord.hpp index 48e92e46c7..e337440731 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.hpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.hpp @@ -54,6 +54,7 @@ class LogRecord bool unset_key(FilterXObject *key); bool is_key_set(FilterXObject *key); uint64_t len() const; + bool iter(FilterXDictIterFunc func, void *user_data); const opentelemetry::proto::logs::v1::LogRecord &get_value() const; private: FilterXOtelLogRecord *super; diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index e744161c79..3af5510946 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -258,6 +258,45 @@ Test(otel_filterx, logrecord_len_and_unset_and_is_key_set) filterx_object_unref(logrecord); } +static gboolean +_append_to_str(FilterXObject *key, FilterXObject *value, gpointer user_data) +{ + GString *output = (GString *) user_data; + + LogMessageValueType type; + filterx_object_marshal_append(key, output, &type); + g_string_append_c(output, '\n'); + filterx_object_marshal_append(value, output, &type); + g_string_append_c(output, '\n'); + + return TRUE; +} + +Test(otel_filterx, logrecord_iter) +{ + FilterXObject *logrecord = filterx_otel_logrecord_new_from_args(NULL); + FilterXObject *body = filterx_string_new("body", -1); + FilterXObject *body_val = filterx_string_new("body_val", -1); + FilterXObject *time_unix_nano = filterx_string_new("time_unix_nano", -1); + FilterXObject *time_unix_nano_val = filterx_integer_new(123); + + cr_assert(filterx_object_set_subscript(logrecord, body, &body_val)); + cr_assert(filterx_object_set_subscript(logrecord, time_unix_nano, &time_unix_nano_val)); + + GString *output = g_string_new(NULL); + cr_assert(filterx_dict_iter(logrecord, _append_to_str, output)); + + cr_assert_str_eq(output->str, "time_unix_nano\n0.000123+00:00\nbody\nbody_val\n"); + + g_string_free(output, TRUE); + filterx_object_unref(time_unix_nano); + filterx_object_unref(time_unix_nano_val); + filterx_object_unref(body); + filterx_object_unref(body_val); + filterx_object_unref(logrecord); +} + + /* Resource */ Test(otel_filterx, resource_empty) From 7a968977fa1824317d71332d286949ef0f2e3dbb Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 17:02:26 +0200 Subject: [PATCH 13/22] otel: port Resource to implement the Dict interface Signed-off-by: Attila Szakacs --- .../otel/filterx/object-otel-resource.cpp | 95 ++++++++++--------- .../otel/filterx/object-otel-resource.hpp | 8 +- modules/grpc/otel/tests/test-otel-filterx.cpp | 4 +- 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 6945cad12a..30bab68d7b 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -60,41 +60,41 @@ Resource::marshal(void) } bool -Resource::set_field(const gchar *attribute, FilterXObject **value) +Resource::set_subscript(FilterXObject *key, FilterXObject **value) { try { - ProtoReflectors reflectors(resource, attribute); + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(resource, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + FilterXObject *assoc_object = NULL; - if (!otel_converter_by_field_descriptor(reflectors.fieldDescriptor)->Set(&resource, attribute, *value, &assoc_object)) + if (!converter->Set(&resource, key_str, *value, &assoc_object)) return false; filterx_object_unref(*value); *value = assoc_object; return true; } - catch (const std::invalid_argument &e) + catch(const std::exception &ex) { - msg_error("FilterX: Failed to set OTel Resource field", - evt_tag_str("field_name", attribute), - evt_tag_str("error", e.what())); return false; } } FilterXObject * -Resource::get_field(const gchar *attribute) +Resource::get_subscript(FilterXObject *key) { try { - ProtoReflectors reflectors(resource, attribute); - return otel_converter_by_field_descriptor(reflectors.fieldDescriptor)->Get(&resource, attribute); + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(resource, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->Get(&resource, key_str); } - catch (const std::invalid_argument &e) + catch(const std::exception &ex) { - msg_error("FilterX: Failed to get OTel Resource field", - evt_tag_str("field_name", attribute), - evt_tag_str("error", e.what())); return nullptr; } } @@ -107,26 +107,6 @@ Resource::get_value() const /* C Wrappers */ -FilterXObject * -_filterx_otel_resource_clone(FilterXObject *s) -{ - FilterXOtelResource *self = (FilterXOtelResource *) s; - - FilterXOtelResource *clone = g_new0(FilterXOtelResource, 1); - filterx_object_init_instance(&clone->super, &FILTERX_TYPE_NAME(otel_resource)); - - try - { - clone->cpp = new Resource(*self->cpp, self); - } - catch (const std::runtime_error &) - { - g_assert_not_reached(); - } - - return &clone->super; -} - static void _free(FilterXObject *s) { @@ -137,19 +117,19 @@ _free(FilterXObject *s) } static gboolean -_setattr(FilterXObject *s, FilterXObject *attr, FilterXObject **new_value) +_set_subscript(FilterXDict *s, FilterXObject *key, FilterXObject **new_value) { FilterXOtelResource *self = (FilterXOtelResource *) s; - return self->cpp->set_field(filterx_string_get_value(attr, NULL), new_value); + return self->cpp->set_subscript(key, new_value); } static FilterXObject * -_getattr(FilterXObject *s, FilterXObject *attr) +_get_subscript(FilterXDict *s, FilterXObject *key) { FilterXOtelResource *self = (FilterXOtelResource *) s; - return self->cpp->get_field(filterx_string_get_value(attr, NULL)); + return self->cpp->get_subscript(key); } static gboolean @@ -171,11 +151,40 @@ _marshal(FilterXObject *s, GString *repr, LogMessageValueType *t) return TRUE; } +static void +_init_instance(FilterXOtelResource *self) +{ + filterx_dict_init_instance(&self->super, &FILTERX_TYPE_NAME(otel_resource)); + + self->super.get_subscript = _get_subscript; + self->super.set_subscript = _set_subscript; +} + +FilterXObject * +_filterx_otel_resource_clone(FilterXObject *s) +{ + FilterXOtelResource *self = (FilterXOtelResource *) s; + + FilterXOtelResource *clone = g_new0(FilterXOtelResource, 1); + _init_instance(clone); + + try + { + clone->cpp = new Resource(*self->cpp, self); + } + catch (const std::runtime_error &) + { + g_assert_not_reached(); + } + + return &clone->super.super; +} + FilterXObject * filterx_otel_resource_new_from_args(GPtrArray *args) { FilterXOtelResource *s = g_new0(FilterXOtelResource, 1); - filterx_object_init_instance((FilterXObject *)s, &FILTERX_TYPE_NAME(otel_resource)); + _init_instance(s); try { @@ -189,11 +198,11 @@ filterx_otel_resource_new_from_args(GPtrArray *args) catch (const std::runtime_error &e) { msg_error("FilterX: Failed to create OTel Resource object", evt_tag_str("error", e.what())); - filterx_object_unref(&s->super); + filterx_object_unref(&s->super.super); return NULL; } - return &s->super; + return &s->super.super; } gpointer @@ -202,12 +211,10 @@ grpc_otel_filterx_resource_construct_new(Plugin *self) return (gpointer) &filterx_otel_resource_new_from_args; } -FILTERX_DEFINE_TYPE(otel_resource, FILTERX_TYPE_NAME(object), +FILTERX_DEFINE_TYPE(otel_resource, FILTERX_TYPE_NAME(dict), .is_mutable = TRUE, .marshal = _marshal, .clone = _filterx_otel_resource_clone, .truthy = _truthy, - .getattr = _getattr, - .setattr = _setattr, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-resource.hpp b/modules/grpc/otel/filterx/object-otel-resource.hpp index cabf66b405..fb90d55bcc 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.hpp +++ b/modules/grpc/otel/filterx/object-otel-resource.hpp @@ -26,7 +26,7 @@ #include "syslog-ng.h" #include "compat/cpp-start.h" -#include "filterx/filterx-object.h" +#include "filterx/object-dict-interface.h" #include "object-otel.h" #include "compat/cpp-end.h" @@ -50,8 +50,8 @@ class Resource Resource(Resource &&o) = delete; std::string marshal(); - bool set_field(const gchar *attribute, FilterXObject **value); - FilterXObject *get_field(const gchar *attribute); + bool set_subscript(FilterXObject *key, FilterXObject **value); + FilterXObject *get_subscript(FilterXObject *key); const opentelemetry::proto::resource::v1::Resource &get_value() const; private: @@ -70,7 +70,7 @@ class Resource struct FilterXOtelResource_ { - FilterXObject super; + FilterXDict super; syslogng::grpc::otel::filterx::Resource *cpp; }; diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 3af5510946..5be640d9e1 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -307,7 +307,7 @@ Test(otel_filterx, resource_empty) cr_assert(MessageDifferencer::Equals(opentelemetry::proto::resource::v1::Resource(), filterx_otel_resource->cpp->get_value())); - filterx_object_unref(&filterx_otel_resource->super); + filterx_object_unref(&filterx_otel_resource->super.super); } Test(otel_filterx, resource_from_protobuf) @@ -328,7 +328,7 @@ Test(otel_filterx, resource_from_protobuf) const opentelemetry::proto::resource::v1::Resource &resource_from_filterx = filterx_otel_resource->cpp->get_value(); cr_assert(MessageDifferencer::Equals(resource, resource_from_filterx)); - filterx_object_unref(&filterx_otel_resource->super); + filterx_object_unref(&filterx_otel_resource->super.super); g_ptr_array_free(args, TRUE); } From 0b28f396b0ceb5c0fafc30a30c8d8ade936f7266 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 17:10:17 +0200 Subject: [PATCH 14/22] otel: implement the rest of the Dict interface for Resource Signed-off-by: Attila Szakacs --- .../otel/filterx/object-otel-resource.cpp | 82 +++++++++++++++++++ .../otel/filterx/object-otel-resource.hpp | 4 + modules/grpc/otel/tests/test-otel-filterx.cpp | 45 ++++++++++ 3 files changed, 131 insertions(+) diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 30bab68d7b..3b4a2fd90c 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -99,6 +99,52 @@ Resource::get_subscript(FilterXObject *key) } } +bool +Resource::unset_key(FilterXObject *key) +{ + try + { + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(resource, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->Unset(&resource, key_str); + } + catch(const std::exception &ex) + { + return false; + } +} + +bool +Resource::is_key_set(FilterXObject *key) +{ + try + { + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(resource, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->IsSet(&resource, key_str); + } + catch(const std::exception &ex) + { + return false; + } +} + +uint64_t +Resource::len() const +{ + return get_protobuf_message_set_field_count(resource); +} + +bool +Resource::iter(FilterXDictIterFunc func, void *user_data) +{ + return iter_on_otel_protobuf_message_fields(resource, func, user_data); +} + const opentelemetry::proto::resource::v1::Resource & Resource::get_value() const { @@ -132,6 +178,38 @@ _get_subscript(FilterXDict *s, FilterXObject *key) return self->cpp->get_subscript(key); } +static gboolean +_unset_key(FilterXDict *s, FilterXObject *key) +{ + FilterXOtelResource *self = (FilterXOtelResource *) s; + + return self->cpp->unset_key(key); +} + +static gboolean +_is_key_set(FilterXDict *s, FilterXObject *key) +{ + FilterXOtelResource *self = (FilterXOtelResource *) s; + + return self->cpp->is_key_set(key); +} + +static guint64 +_len(FilterXDict *s) +{ + FilterXOtelResource *self = (FilterXOtelResource *) s; + + return self->cpp->len(); +} + +static gboolean +_iter(FilterXDict *s, FilterXDictIterFunc func, gpointer user_data) +{ + FilterXOtelResource *self = (FilterXOtelResource *) s; + + return self->cpp->iter(func, user_data); +} + static gboolean _truthy(FilterXObject *s) { @@ -158,6 +236,10 @@ _init_instance(FilterXOtelResource *self) self->super.get_subscript = _get_subscript; self->super.set_subscript = _set_subscript; + self->super.unset_key = _unset_key; + self->super.is_key_set = _is_key_set; + self->super.len = _len; + self->super.iter = _iter; } FilterXObject * diff --git a/modules/grpc/otel/filterx/object-otel-resource.hpp b/modules/grpc/otel/filterx/object-otel-resource.hpp index fb90d55bcc..e424e94fde 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.hpp +++ b/modules/grpc/otel/filterx/object-otel-resource.hpp @@ -52,6 +52,10 @@ class Resource std::string marshal(); bool set_subscript(FilterXObject *key, FilterXObject **value); FilterXObject *get_subscript(FilterXObject *key); + bool unset_key(FilterXObject *key); + bool is_key_set(FilterXObject *key); + uint64_t len() const; + bool iter(FilterXDictIterFunc func, void *user_data); const opentelemetry::proto::resource::v1::Resource &get_value() const; private: diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 5be640d9e1..a81b778dc0 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -429,6 +429,51 @@ Test(otel_filterx, resource_set_field) filterx_object_unref(filterx_otel_resource); } +Test(otel_filterx, resource_len_and_unset_and_is_key_set) +{ + FilterXObject *resource = filterx_otel_resource_new_from_args(NULL); + FilterXObject *dropped_attributes_count = filterx_string_new("dropped_attributes_count", -1); + FilterXObject *dropped_attributes_count_val = filterx_integer_new(42); + + guint64 len; + cr_assert(filterx_object_len(resource, &len)); + cr_assert_eq(len, 0); + + cr_assert_not(filterx_object_is_key_set(resource, dropped_attributes_count)); + cr_assert(filterx_object_set_subscript(resource, dropped_attributes_count, &dropped_attributes_count_val)); + cr_assert(filterx_object_len(resource, &len)); + cr_assert_eq(len, 1); + cr_assert(filterx_object_is_key_set(resource, dropped_attributes_count)); + + cr_assert(filterx_object_unset_key(resource, dropped_attributes_count)); + cr_assert(filterx_object_len(resource, &len)); + cr_assert_eq(len, 0); + cr_assert_not(filterx_object_is_key_set(resource, dropped_attributes_count)); + + filterx_object_unref(dropped_attributes_count); + filterx_object_unref(dropped_attributes_count_val); + filterx_object_unref(resource); +} + +Test(otel_filterx, resource_iter) +{ + FilterXObject *resource = filterx_otel_resource_new_from_args(NULL); + FilterXObject *dropped_attributes_count = filterx_string_new("dropped_attributes_count", -1); + FilterXObject *dropped_attributes_count_val = filterx_integer_new(42); + + cr_assert(filterx_object_set_subscript(resource, dropped_attributes_count, &dropped_attributes_count_val)); + + GString *output = g_string_new(NULL); + cr_assert(filterx_dict_iter(resource, _append_to_str, output)); + + cr_assert_str_eq(output->str, "dropped_attributes_count\n42\n"); + + g_string_free(output, TRUE); + filterx_object_unref(dropped_attributes_count); + filterx_object_unref(dropped_attributes_count_val); + filterx_object_unref(resource); +} + /* Scope */ From fe4773e8e31f3eb5d5acbbaddf819fff0068bbb2 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 17:18:11 +0200 Subject: [PATCH 15/22] otel: port Scope to implement the Dict interface Signed-off-by: Attila Szakacs --- .../grpc/otel/filterx/object-otel-scope.cpp | 95 ++++++++++--------- .../grpc/otel/filterx/object-otel-scope.hpp | 8 +- modules/grpc/otel/tests/test-otel-filterx.cpp | 4 +- 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index 6313dce9fd..5e9eb2c21f 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -60,41 +60,41 @@ Scope::marshal(void) } bool -Scope::set_field(const gchar *attribute, FilterXObject **value) +Scope::set_subscript(FilterXObject *key, FilterXObject **value) { try { - ProtoReflectors reflectors(scope, attribute); + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(scope, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + FilterXObject *assoc_object = NULL; - if (!otel_converter_by_field_descriptor(reflectors.fieldDescriptor)->Set(&scope, attribute, *value, &assoc_object)) + if (!converter->Set(&scope, key_str, *value, &assoc_object)) return false; filterx_object_unref(*value); *value = assoc_object; return true; } - catch (const std::invalid_argument &e) + catch(const std::exception &ex) { - msg_error("FilterX: Failed to set OTel Scope field", - evt_tag_str("field_name", attribute), - evt_tag_str("error", e.what())); return false; } } FilterXObject * -Scope::get_field(const gchar *attribute) +Scope::get_subscript(FilterXObject *key) { try { - ProtoReflectors reflectors(scope, attribute); - return otel_converter_by_field_descriptor(reflectors.fieldDescriptor)->Get(&scope, attribute); + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(scope, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->Get(&scope, key_str); } - catch (const std::invalid_argument &e) + catch(const std::exception &ex) { - msg_error("FilterX: Failed to get OTel Scope field", - evt_tag_str("field_name", attribute), - evt_tag_str("error", e.what())); return nullptr; } } @@ -107,26 +107,6 @@ Scope::get_value() const /* C Wrappers */ -FilterXObject * -_filterx_otel_scope_clone(FilterXObject *s) -{ - FilterXOtelScope *self = (FilterXOtelScope *) s; - - FilterXOtelScope *clone = g_new0(FilterXOtelScope, 1); - filterx_object_init_instance(&clone->super, &FILTERX_TYPE_NAME(otel_scope)); - - try - { - clone->cpp = new Scope(*self->cpp, self); - } - catch (const std::runtime_error &) - { - g_assert_not_reached(); - } - - return &clone->super; -} - static void _free(FilterXObject *s) { @@ -137,19 +117,19 @@ _free(FilterXObject *s) } static gboolean -_setattr(FilterXObject *s, FilterXObject *attr, FilterXObject **new_value) +_set_subscript(FilterXDict *s, FilterXObject *key, FilterXObject **new_value) { FilterXOtelScope *self = (FilterXOtelScope *) s; - return self->cpp->set_field(filterx_string_get_value(attr, NULL), new_value); + return self->cpp->set_subscript(key, new_value); } static FilterXObject * -_getattr(FilterXObject *s, FilterXObject *attr) +_get_subscript(FilterXDict *s, FilterXObject *key) { FilterXOtelScope *self = (FilterXOtelScope *) s; - return self->cpp->get_field(filterx_string_get_value(attr, NULL)); + return self->cpp->get_subscript(key); } static gboolean @@ -171,11 +151,40 @@ _marshal(FilterXObject *s, GString *repr, LogMessageValueType *t) return TRUE; } +static void +_init_instance(FilterXOtelScope *self) +{ + filterx_dict_init_instance(&self->super, &FILTERX_TYPE_NAME(otel_scope)); + + self->super.set_subscript = _set_subscript; + self->super.get_subscript = _get_subscript; +} + +FilterXObject * +_filterx_otel_scope_clone(FilterXObject *s) +{ + FilterXOtelScope *self = (FilterXOtelScope *) s; + + FilterXOtelScope *clone = g_new0(FilterXOtelScope, 1); + _init_instance(clone); + + try + { + clone->cpp = new Scope(*self->cpp, self); + } + catch (const std::runtime_error &) + { + g_assert_not_reached(); + } + + return &clone->super.super; +} + FilterXObject * filterx_otel_scope_new_from_args(GPtrArray *args) { FilterXOtelScope *s = g_new0(FilterXOtelScope, 1); - filterx_object_init_instance((FilterXObject *)s, &FILTERX_TYPE_NAME(otel_scope)); + _init_instance(s); try { @@ -189,11 +198,11 @@ filterx_otel_scope_new_from_args(GPtrArray *args) catch (const std::runtime_error &e) { msg_error("FilterX: Failed to create OTel Scope object", evt_tag_str("error", e.what())); - filterx_object_unref(&s->super); + filterx_object_unref(&s->super.super); return NULL; } - return &s->super; + return &s->super.super; } gpointer @@ -202,12 +211,10 @@ grpc_otel_filterx_scope_construct_new(Plugin *self) return (gpointer) &filterx_otel_scope_new_from_args; } -FILTERX_DEFINE_TYPE(otel_scope, FILTERX_TYPE_NAME(object), +FILTERX_DEFINE_TYPE(otel_scope, FILTERX_TYPE_NAME(dict), .is_mutable = TRUE, .marshal = _marshal, .clone = _filterx_otel_scope_clone, .truthy = _truthy, - .getattr = _getattr, - .setattr = _setattr, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-scope.hpp b/modules/grpc/otel/filterx/object-otel-scope.hpp index 62dc3a9876..e9bcf04e23 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.hpp +++ b/modules/grpc/otel/filterx/object-otel-scope.hpp @@ -26,7 +26,7 @@ #include "syslog-ng.h" #include "compat/cpp-start.h" -#include "filterx/filterx-object.h" +#include "filterx/object-dict-interface.h" #include "object-otel.h" #include "compat/cpp-end.h" @@ -50,8 +50,8 @@ class Scope Scope(Scope &&o) = delete; std::string marshal(); - bool set_field(const gchar *attribute, FilterXObject **value); - FilterXObject *get_field(const gchar *attribute); + bool set_subscript(FilterXObject *key, FilterXObject **value); + FilterXObject *get_subscript(FilterXObject *key); const opentelemetry::proto::common::v1::InstrumentationScope &get_value() const; private: @@ -70,7 +70,7 @@ class Scope struct FilterXOtelScope_ { - FilterXObject super; + FilterXDict super; syslogng::grpc::otel::filterx::Scope *cpp; }; diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index a81b778dc0..8c78c71314 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -485,7 +485,7 @@ Test(otel_filterx, scope_empty) cr_assert(MessageDifferencer::Equals(opentelemetry::proto::common::v1::InstrumentationScope(), filterx_otel_scope->cpp->get_value())); - filterx_object_unref(&filterx_otel_scope->super); + filterx_object_unref(&filterx_otel_scope->super.super); } Test(otel_filterx, scope_from_protobuf) @@ -508,7 +508,7 @@ Test(otel_filterx, scope_from_protobuf) const opentelemetry::proto::common::v1::InstrumentationScope &scope_from_filterx = filterx_otel_scope->cpp->get_value(); cr_assert(MessageDifferencer::Equals(scope, scope_from_filterx)); - filterx_object_unref(&filterx_otel_scope->super); + filterx_object_unref(&filterx_otel_scope->super.super); g_ptr_array_free(args, TRUE); } From 224c2e2652b1d10e9ecbea6d7426325feed9fee5 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 7 May 2024 17:22:13 +0200 Subject: [PATCH 16/22] otel: implement the rest of the Dict interface for Scope Signed-off-by: Attila Szakacs --- .../grpc/otel/filterx/object-otel-scope.cpp | 82 +++++++++++++++++++ .../grpc/otel/filterx/object-otel-scope.hpp | 4 + modules/grpc/otel/tests/test-otel-filterx.cpp | 66 +++++++++++++++ 3 files changed, 152 insertions(+) diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index 5e9eb2c21f..30e259046a 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -99,6 +99,52 @@ Scope::get_subscript(FilterXObject *key) } } +bool +Scope::unset_key(FilterXObject *key) +{ + try + { + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(scope, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->Unset(&scope, key_str); + } + catch(const std::exception &ex) + { + return false; + } +} + +bool +Scope::is_key_set(FilterXObject *key) +{ + try + { + std::string key_str = extract_string_from_object(key); + ProtoReflectors reflectors(scope, key_str); + ProtobufField *converter = otel_converter_by_field_descriptor(reflectors.fieldDescriptor); + + return converter->IsSet(&scope, key_str); + } + catch(const std::exception &ex) + { + return false; + } +} + +uint64_t +Scope::len() const +{ + return get_protobuf_message_set_field_count(scope); +} + +bool +Scope::iter(FilterXDictIterFunc func, void *user_data) +{ + return iter_on_otel_protobuf_message_fields(scope, func, user_data); +} + const opentelemetry::proto::common::v1::InstrumentationScope & Scope::get_value() const { @@ -132,6 +178,38 @@ _get_subscript(FilterXDict *s, FilterXObject *key) return self->cpp->get_subscript(key); } +static gboolean +_unset_key(FilterXDict *s, FilterXObject *key) +{ + FilterXOtelScope *self = (FilterXOtelScope *) s; + + return self->cpp->unset_key(key); +} + +static gboolean +_is_key_set(FilterXDict *s, FilterXObject *key) +{ + FilterXOtelScope *self = (FilterXOtelScope *) s; + + return self->cpp->is_key_set(key); +} + +static guint64 +_len(FilterXDict *s) +{ + FilterXOtelScope *self = (FilterXOtelScope *) s; + + return self->cpp->len(); +} + +static gboolean +_iter(FilterXDict *s, FilterXDictIterFunc func, gpointer user_data) +{ + FilterXOtelScope *self = (FilterXOtelScope *) s; + + return self->cpp->iter(func, user_data); +} + static gboolean _truthy(FilterXObject *s) { @@ -158,6 +236,10 @@ _init_instance(FilterXOtelScope *self) self->super.set_subscript = _set_subscript; self->super.get_subscript = _get_subscript; + self->super.unset_key = _unset_key; + self->super.is_key_set = _is_key_set; + self->super.len = _len; + self->super.iter = _iter; } FilterXObject * diff --git a/modules/grpc/otel/filterx/object-otel-scope.hpp b/modules/grpc/otel/filterx/object-otel-scope.hpp index e9bcf04e23..306c3ecc47 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.hpp +++ b/modules/grpc/otel/filterx/object-otel-scope.hpp @@ -52,6 +52,10 @@ class Scope std::string marshal(); bool set_subscript(FilterXObject *key, FilterXObject **value); FilterXObject *get_subscript(FilterXObject *key); + bool unset_key(FilterXObject *key); + bool is_key_set(FilterXObject *key); + uint64_t len() const; + bool iter(FilterXDictIterFunc func, void *user_data); const opentelemetry::proto::common::v1::InstrumentationScope &get_value() const; private: diff --git a/modules/grpc/otel/tests/test-otel-filterx.cpp b/modules/grpc/otel/tests/test-otel-filterx.cpp index 8c78c71314..7d17542181 100644 --- a/modules/grpc/otel/tests/test-otel-filterx.cpp +++ b/modules/grpc/otel/tests/test-otel-filterx.cpp @@ -881,6 +881,72 @@ Test(otel_filterx, kvlist_through_logrecord) filterx_object_unref(fx_logrecord); } +Test(otel_filterx, scope_len_and_unset_and_is_key_set) +{ + FilterXObject *scope = filterx_otel_scope_new_from_args(NULL); + FilterXObject *name = filterx_string_new("name", -1); + FilterXObject *name_val = filterx_string_new("name_val", -1); + FilterXObject *version = filterx_string_new("version", -1); + FilterXObject *version_val = filterx_string_new("version_val", -1); + + guint64 len; + cr_assert(filterx_object_len(scope, &len)); + cr_assert_eq(len, 0); + + cr_assert_not(filterx_object_is_key_set(scope, name)); + cr_assert(filterx_object_set_subscript(scope, name, &name_val)); + cr_assert(filterx_object_len(scope, &len)); + cr_assert_eq(len, 1); + cr_assert(filterx_object_is_key_set(scope, name)); + + cr_assert_not(filterx_object_is_key_set(scope, version)); + cr_assert(filterx_object_set_subscript(scope, version, &version_val)); + cr_assert(filterx_object_len(scope, &len)); + cr_assert_eq(len, 2); + cr_assert(filterx_object_is_key_set(scope, version)); + + cr_assert(filterx_object_unset_key(scope, name)); + cr_assert(filterx_object_len(scope, &len)); + cr_assert_eq(len, 1); + cr_assert_not(filterx_object_is_key_set(scope, name)); + + cr_assert(filterx_object_unset_key(scope, version)); + cr_assert(filterx_object_len(scope, &len)); + cr_assert_eq(len, 0); + cr_assert_not(filterx_object_is_key_set(scope, version)); + + filterx_object_unref(version); + filterx_object_unref(version_val); + filterx_object_unref(name); + filterx_object_unref(name_val); + filterx_object_unref(scope); +} + +Test(otel_filterx, scope_iter) +{ + FilterXObject *scope = filterx_otel_scope_new_from_args(NULL); + FilterXObject *name = filterx_string_new("name", -1); + FilterXObject *name_val = filterx_string_new("name_val", -1); + FilterXObject *version = filterx_string_new("version", -1); + FilterXObject *version_val = filterx_string_new("version_val", -1); + + cr_assert(filterx_object_set_subscript(scope, name, &name_val)); + cr_assert(filterx_object_set_subscript(scope, version, &version_val)); + + GString *output = g_string_new(NULL); + cr_assert(filterx_dict_iter(scope, _append_to_str, output)); + + cr_assert_str_eq(output->str, "name\nname_val\nversion\nversion_val\n"); + + g_string_free(output, TRUE); + filterx_object_unref(version); + filterx_object_unref(version_val); + filterx_object_unref(name); + filterx_object_unref(name_val); + filterx_object_unref(scope); +} + + /* Array */ Test(otel_filterx, array_empty) From 641035d52f64d721fab3302e044314bce396942b Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Wed, 8 May 2024 09:38:16 +0200 Subject: [PATCH 17/22] otel: add list and dict factory for Resource, Scope, LogRecord Signed-off-by: Attila Szakacs --- modules/grpc/otel/filterx/object-otel-logrecord.cpp | 2 ++ modules/grpc/otel/filterx/object-otel-resource.cpp | 2 ++ modules/grpc/otel/filterx/object-otel-scope.cpp | 2 ++ 3 files changed, 6 insertions(+) diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 720d7cfa0f..52d256de23 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -311,5 +311,7 @@ FILTERX_DEFINE_TYPE(otel_logrecord, FILTERX_TYPE_NAME(dict), .marshal = _marshal, .clone = _filterx_otel_logrecord_clone, .truthy = _truthy, + .list_factory = filterx_otel_array_new, + .dict_factory = filterx_otel_kvlist_new, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 3b4a2fd90c..4de79a034b 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -298,5 +298,7 @@ FILTERX_DEFINE_TYPE(otel_resource, FILTERX_TYPE_NAME(dict), .marshal = _marshal, .clone = _filterx_otel_resource_clone, .truthy = _truthy, + .list_factory = filterx_otel_array_new, + .dict_factory = filterx_otel_kvlist_new, .free_fn = _free, ); diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index 30e259046a..075f448ede 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -298,5 +298,7 @@ FILTERX_DEFINE_TYPE(otel_scope, FILTERX_TYPE_NAME(dict), .marshal = _marshal, .clone = _filterx_otel_scope_clone, .truthy = _truthy, + .list_factory = filterx_otel_array_new, + .dict_factory = filterx_otel_kvlist_new, .free_fn = _free, ); From a3a11ad0374073d9be1b700bf5647cc04021bedb Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Wed, 8 May 2024 09:52:36 +0200 Subject: [PATCH 18/22] filterx: add filterx_dict_merge() Signed-off-by: Attila Szakacs --- lib/filterx/object-dict-interface.c | 21 +++++++++++++++++++++ lib/filterx/object-dict-interface.h | 1 + 2 files changed, 22 insertions(+) diff --git a/lib/filterx/object-dict-interface.c b/lib/filterx/object-dict-interface.c index b6650b0189..4fff5614e3 100644 --- a/lib/filterx/object-dict-interface.c +++ b/lib/filterx/object-dict-interface.c @@ -34,6 +34,27 @@ filterx_dict_iter(FilterXObject *s, FilterXDictIterFunc func, gpointer user_data return self->iter(self, func, user_data); } +static gboolean +_add_elem_to_dict(FilterXObject *key_obj, FilterXObject *value_obj, gpointer user_data) +{ + FilterXObject *dict = (FilterXObject *) user_data; + + FilterXObject *new_value = filterx_object_ref(value_obj); + gboolean success = filterx_object_set_subscript(dict, key_obj, &new_value); + filterx_object_unref(new_value); + + return success; +} + +gboolean +filterx_dict_merge(FilterXObject *s, FilterXObject *other) +{ + FilterXDict *self = (FilterXDict *) s; + + g_assert(filterx_object_is_type(other, &FILTERX_TYPE_NAME(dict))); + return filterx_dict_iter(other, _add_elem_to_dict, self); +} + static gboolean _len(FilterXObject *s, guint64 *len) { diff --git a/lib/filterx/object-dict-interface.h b/lib/filterx/object-dict-interface.h index 023703b5b2..87dd74d0a7 100644 --- a/lib/filterx/object-dict-interface.h +++ b/lib/filterx/object-dict-interface.h @@ -44,6 +44,7 @@ struct FilterXDict_ }; gboolean filterx_dict_iter(FilterXObject *s, FilterXDictIterFunc func, gpointer user_data); +gboolean filterx_dict_merge(FilterXObject *s, FilterXObject *other); void filterx_dict_init_instance(FilterXDict *self, FilterXType *type); From 003df128ff3e146a7b0427186bf3530bc421a0f0 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Wed, 8 May 2024 10:12:33 +0200 Subject: [PATCH 19/22] filterx: add filterx_list_merge() Signed-off-by: Attila Szakacs --- lib/filterx/object-list-interface.c | 22 ++++++++++++++++++++++ lib/filterx/object-list-interface.h | 1 + 2 files changed, 23 insertions(+) diff --git a/lib/filterx/object-list-interface.c b/lib/filterx/object-list-interface.c index 7ba00b4e3b..bd759a915c 100644 --- a/lib/filterx/object-list-interface.c +++ b/lib/filterx/object-list-interface.c @@ -61,6 +61,28 @@ filterx_list_unset_index(FilterXObject *s, gint64 index) return result; } +gboolean +filterx_list_merge(FilterXObject *s, FilterXObject *other) +{ + g_assert(filterx_object_is_type(other, &FILTERX_TYPE_NAME(list))); + + guint64 len; + g_assert(filterx_object_len(other, &len)); + + for (guint64 i = 0; i < len; i++) + { + FilterXObject *value_obj = filterx_list_get_subscript(other, (gint64) MIN(i, G_MAXINT64)); + gboolean success = filterx_list_append(s, &value_obj); + + filterx_object_unref(value_obj); + + if (!success) + return FALSE; + } + + return TRUE; +} + static gboolean _len(FilterXObject *s, guint64 *len) { diff --git a/lib/filterx/object-list-interface.h b/lib/filterx/object-list-interface.h index f5c5ea74cf..d4a2725ba5 100644 --- a/lib/filterx/object-list-interface.h +++ b/lib/filterx/object-list-interface.h @@ -43,6 +43,7 @@ FilterXObject *filterx_list_get_subscript(FilterXObject *s, gint64 index); gboolean filterx_list_set_subscript(FilterXObject *s, gint64 index, FilterXObject **new_value); gboolean filterx_list_append(FilterXObject *s, FilterXObject **new_value); gboolean filterx_list_unset_index(FilterXObject *s, gint64 index); +gboolean filterx_list_merge(FilterXObject *s, FilterXObject *other); void filterx_list_init_instance(FilterXList *self, FilterXType *type); From 6af1e67376191c5ce1ed39e3537867b9d5b3f0d5 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Wed, 8 May 2024 10:17:13 +0200 Subject: [PATCH 20/22] filterx: use dict merging in json and otel related functions Signed-off-by: Attila Szakacs --- lib/filterx/object-json.c | 12 ++++++++++ .../grpc/otel/filterx/object-otel-kvlist.cpp | 22 ++++++++++++++++--- .../otel/filterx/object-otel-logrecord.cpp | 22 ++++++++++++++++--- .../otel/filterx/object-otel-resource.cpp | 22 ++++++++++++++++--- .../grpc/otel/filterx/object-otel-scope.cpp | 22 ++++++++++++++++--- 5 files changed, 88 insertions(+), 12 deletions(-) diff --git a/lib/filterx/object-json.c b/lib/filterx/object-json.c index 3b47bfad53..b847bbaaed 100644 --- a/lib/filterx/object-json.c +++ b/lib/filterx/object-json.c @@ -24,6 +24,7 @@ #include "filterx/object-null.h" #include "filterx/object-primitive.h" #include "filterx/object-string.h" +#include "filterx/object-dict-interface.h" #include "filterx/object-message-value.h" #include "filterx/filterx-weakrefs.h" #include "filterx/filterx-eval.h" @@ -167,6 +168,17 @@ filterx_json_new_from_args(GPtrArray *args) filterx_object_is_type(arg, &FILTERX_TYPE_NAME(json_object))) return filterx_object_ref(arg); + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + { + FilterXObject *self = filterx_json_object_new_empty(); + if (!filterx_dict_merge(self, arg)) + { + filterx_object_unref(self); + return NULL; + } + return self; + } + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(message_value))) { FilterXObject *unmarshalled = filterx_object_unmarshal(arg); diff --git a/modules/grpc/otel/filterx/object-otel-kvlist.cpp b/modules/grpc/otel/filterx/object-otel-kvlist.cpp index 8b7edf2fb5..33b9c80171 100644 --- a/modules/grpc/otel/filterx/object-otel-kvlist.cpp +++ b/modules/grpc/otel/filterx/object-otel-kvlist.cpp @@ -354,11 +354,27 @@ filterx_otel_kvlist_new_from_args(GPtrArray *args) try { if (!args || args->len == 0) - self->cpp = new KVList(self); + { + self->cpp = new KVList(self); + } else if (args->len == 1) - self->cpp = new KVList(self, (FilterXObject *) g_ptr_array_index(args, 0)); + { + FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + { + self->cpp = new KVList(self); + if (!filterx_dict_merge(&self->super.super, arg)) + throw std::runtime_error("Failed to merge dict"); + } + else + { + self->cpp = new KVList(self, arg); + } + } else - throw std::runtime_error("Invalid number of arguments"); + { + throw std::runtime_error("Invalid number of arguments"); + } } catch (const std::runtime_error &e) { diff --git a/modules/grpc/otel/filterx/object-otel-logrecord.cpp b/modules/grpc/otel/filterx/object-otel-logrecord.cpp index 52d256de23..e2b961b4ab 100644 --- a/modules/grpc/otel/filterx/object-otel-logrecord.cpp +++ b/modules/grpc/otel/filterx/object-otel-logrecord.cpp @@ -290,11 +290,27 @@ filterx_otel_logrecord_new_from_args(GPtrArray *args) try { if (!args || args->len == 0) - self->cpp = new LogRecord(self); + { + self->cpp = new LogRecord(self); + } else if (args->len == 1) - self->cpp = new LogRecord(self, (FilterXObject *) g_ptr_array_index(args, 0)); + { + FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + { + self->cpp = new LogRecord(self); + if (!filterx_dict_merge(&self->super.super, arg)) + throw std::runtime_error("Failed to merge dict"); + } + else + { + self->cpp = new LogRecord(self, arg); + } + } else - throw std::runtime_error("Invalid number of arguments"); + { + throw std::runtime_error("Invalid number of arguments"); + } } catch (const std::runtime_error &e) { diff --git a/modules/grpc/otel/filterx/object-otel-resource.cpp b/modules/grpc/otel/filterx/object-otel-resource.cpp index 4de79a034b..a7847a7a32 100644 --- a/modules/grpc/otel/filterx/object-otel-resource.cpp +++ b/modules/grpc/otel/filterx/object-otel-resource.cpp @@ -271,11 +271,27 @@ filterx_otel_resource_new_from_args(GPtrArray *args) try { if (!args || args->len == 0) - s->cpp = new Resource(s); + { + s->cpp = new Resource(s); + } else if (args->len == 1) - s->cpp = new Resource(s, (FilterXObject *) g_ptr_array_index(args, 0)); + { + FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + { + s->cpp = new Resource(s); + if (!filterx_dict_merge(&s->super.super, arg)) + throw std::runtime_error("Failed to merge dict"); + } + else + { + s->cpp = new Resource(s, arg); + } + } else - throw std::runtime_error("Invalid number of arguments"); + { + throw std::runtime_error("Invalid number of arguments"); + } } catch (const std::runtime_error &e) { diff --git a/modules/grpc/otel/filterx/object-otel-scope.cpp b/modules/grpc/otel/filterx/object-otel-scope.cpp index 075f448ede..6904117c69 100644 --- a/modules/grpc/otel/filterx/object-otel-scope.cpp +++ b/modules/grpc/otel/filterx/object-otel-scope.cpp @@ -271,11 +271,27 @@ filterx_otel_scope_new_from_args(GPtrArray *args) try { if (!args || args->len == 0) - s->cpp = new Scope(s); + { + s->cpp = new Scope(s); + } else if (args->len == 1) - s->cpp = new Scope(s, (FilterXObject *) g_ptr_array_index(args, 0)); + { + FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(dict))) + { + s->cpp = new Scope(s); + if (!filterx_dict_merge(&s->super.super, arg)) + throw std::runtime_error("Failed to merge dict"); + } + else + { + s->cpp = new Scope(s, arg); + } + } else - throw std::runtime_error("Invalid number of arguments"); + { + throw std::runtime_error("Invalid number of arguments"); + } } catch (const std::runtime_error &e) { From 46ed4e8a07a4d00b52c5d72a16bf9142312830f1 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Wed, 8 May 2024 10:18:37 +0200 Subject: [PATCH 21/22] filterx: use list merging in json and otel related functions Signed-off-by: Attila Szakacs --- lib/filterx/object-json.c | 12 ++++++++++ .../grpc/otel/filterx/object-otel-array.cpp | 22 ++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/filterx/object-json.c b/lib/filterx/object-json.c index b847bbaaed..137bca01ec 100644 --- a/lib/filterx/object-json.c +++ b/lib/filterx/object-json.c @@ -25,6 +25,7 @@ #include "filterx/object-primitive.h" #include "filterx/object-string.h" #include "filterx/object-dict-interface.h" +#include "filterx/object-list-interface.h" #include "filterx/object-message-value.h" #include "filterx/filterx-weakrefs.h" #include "filterx/filterx-eval.h" @@ -179,6 +180,17 @@ filterx_json_new_from_args(GPtrArray *args) return self; } + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(list))) + { + FilterXObject *self = filterx_json_array_new_empty(); + if (!filterx_list_merge(self, arg)) + { + filterx_object_unref(self); + return NULL; + } + return self; + } + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(message_value))) { FilterXObject *unmarshalled = filterx_object_unmarshal(arg); diff --git a/modules/grpc/otel/filterx/object-otel-array.cpp b/modules/grpc/otel/filterx/object-otel-array.cpp index fd87aba5d5..7dbe683352 100644 --- a/modules/grpc/otel/filterx/object-otel-array.cpp +++ b/modules/grpc/otel/filterx/object-otel-array.cpp @@ -252,11 +252,27 @@ filterx_otel_array_new_from_args(GPtrArray *args) try { if (!args || args->len == 0) - self->cpp = new Array(self); + { + self->cpp = new Array(self); + } else if (args->len == 1) - self->cpp = new Array(self, (FilterXObject *) g_ptr_array_index(args, 0)); + { + FilterXObject *arg = (FilterXObject *) g_ptr_array_index(args, 0); + if (filterx_object_is_type(arg, &FILTERX_TYPE_NAME(list))) + { + self->cpp = new Array(self); + if (!filterx_list_merge(&self->super.super, arg)) + throw std::runtime_error("Failed to merge list"); + } + else + { + self->cpp = new Array(self, arg); + } + } else - throw std::runtime_error("Invalid number of arguments"); + { + throw std::runtime_error("Invalid number of arguments"); + } } catch (const std::runtime_error &e) { From 7808171a3c1cd6709eb9a454354a28e81a179521 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Wed, 8 May 2024 10:47:14 +0200 Subject: [PATCH 22/22] filterx: add E2E tests to OTEL Res, Sco, Log <-> JSON convert Signed-off-by: Attila Szakacs --- .../functional_tests/filterx/test_filterx.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index 0479bd0b59..4d88316f0a 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -330,6 +330,153 @@ def test_otel_to_json(config, syslog_ng): assert file_true.read_log() == '{"otel_kvl":{"foo":42,"bar":1337},"otel_arr":[1,2,3,4]}\n' +def test_otel_resource_scope_log_to_json(config, syslog_ng): + (file_true, file_false) = create_config( + config, r""" + log = otel_logrecord(); + log += { + "body": "fit", + "time_unix_nano": 123456789, + "attributes": { + "answer": 42, + "cool": true + } + }; + + resource = otel_resource(); + resource.attributes["host.name"] = "szami.to"; + + scope = otel_scope({ + "name": "oszcillo", + "version": "ceta" + }); + + resource_js = json(resource); + scope_js = json(scope); + log_js = json(log); + + js = json({ + "in_dict_generator": { + "resource": resource, + "scope": scope, + "log": log, + }, + "from_json_cast": { + "resource": resource_js, + "scope": scope_js, + "log": log_js, + } + }); + + $MSG = js; + """, + ) + syslog_ng.start(config) + + assert file_true.get_stats()["processed"] == 1 + assert "processed" not in file_false.get_stats() + assert json.loads(file_true.read_log()) == { + "in_dict_generator": { + "resource": { + "attributes": { + "host.name": "szami.to", + }, + }, + "scope": { + "name": "oszcillo", + "version": "ceta", + }, + "log": { + "time_unix_nano": "123.456789+00:00", + "body": "fit", + "attributes": { + "answer": 42, + "cool": True, + }, + }, + }, + "from_json_cast": { + "resource": { + "attributes": { + "host.name": "szami.to", + }, + }, + "scope": { + "name": "oszcillo", + "version": "ceta", + }, + "log": { + "time_unix_nano": "123.456789+00:00", + "body": "fit", + "attributes": { + "answer": 42, + "cool": True, + }, + }, + }, + } + + +def test_json_to_otel_resource_scope_log(config, syslog_ng): + (file_true, file_false) = create_config( + config, r""" + log_js = json(); + log_js += { + "body": "fit", + "time_unix_nano": 123456789, + "attributes": { + "answer": 42, + "cool": true + } + }; + + resource_js = json(); + resource_js.attributes = {}; + resource_js.attributes["host.name"] = "szami.to"; + + scope_js = json({ + "name": "oszcillo", + "version": "ceta" + }); + + resource = otel_resource(resource_js); + scope = otel_scope(scope_js); + log = otel_logrecord(log_js); + + js = json({ + "resource": resource, + "scope": scope, + "log": log, + }); + + $MSG = js; + """, + ) + syslog_ng.start(config) + + assert file_true.get_stats()["processed"] == 1 + assert "processed" not in file_false.get_stats() + assert json.loads(file_true.read_log()) == { + "resource": { + "attributes": { + "host.name": "szami.to", + }, + }, + "scope": { + "name": "oszcillo", + "version": "ceta", + }, + "log": { + "time_unix_nano": "123.456789+00:00", + "body": "fit", + "attributes": { + "answer": 42, + "cool": True, + }, + }, + } + + def test_simple_true_condition(config, syslog_ng): (file_true, file_false) = create_config(config, """ true; """) syslog_ng.start(config)