From 8b6f37b75eec09c32f0dbca223080cf9686e6a9b Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Tue, 9 Jan 2024 16:20:53 -0500 Subject: [PATCH 1/2] feat: Inline contexts for all evaluation events (#245) --- contract-tests/service.py | 3 ++- ldclient/impl/events/event_processor.py | 2 +- testing/impl/events/test_event_processor.py | 19 ++++++++----------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/contract-tests/service.py b/contract-tests/service.py index 18345ce..fea8b26 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -71,7 +71,8 @@ def status(): 'tags', 'migrations', 'event-sampling', - 'polling-gzip' + 'polling-gzip', + 'inline-context' ] } return (json.dumps(body), 200, {'Content-type': 'application/json'}) diff --git a/ldclient/impl/events/event_processor.py b/ldclient/impl/events/event_processor.py index 81a276a..1556238 100644 --- a/ldclient/impl/events/event_processor.py +++ b/ldclient/impl/events/event_processor.py @@ -65,7 +65,7 @@ def make_output_events(self, events: List[Any], summary: EventSummary): def make_output_event(self, e: Any): if isinstance(e, EventInputEvaluation): out = self._base_eval_props(e, 'feature') - out['contextKeys'] = self._context_keys(e.context) + out['context'] = self._process_context(e.context) return out elif isinstance(e, DebugEvent): out = self._base_eval_props(e.original_input, 'debug') diff --git a/testing/impl/events/test_event_processor.py b/testing/impl/events/test_event_processor.py index b2bf91d..908ca3e 100644 --- a/testing/impl/events/test_event_processor.py +++ b/testing/impl/events/test_event_processor.py @@ -15,6 +15,7 @@ from ldclient.migrations.tracker import MigrationOpEvent from ldclient.impl.events.types import EventInput, EventInputCustom, EventInputEvaluation, EventInputIdentify from ldclient.impl.util import timedelta_millis +from ldclient.impl.events.event_context_formatter import EventContextFormatter from testing.builders import * from testing.proxy_test_util import do_proxy_tests @@ -23,12 +24,6 @@ default_config = Config("fake_sdk_key") context = Context.builder('userkey').name('Red').build() -filtered_context = context.to_dict() # TODO: implement attribute redaction -filtered_context = { - 'kind': 'user', - 'key': 'userkey', - '_meta': {'redactedAttributes': ['name']} -} flag = FlagBuilder('flagkey').version(2).build() flag_with_0_sampling_ratio = FlagBuilder('flagkey').version(3).sampling_ratio(0).build() flag_excluded_from_summaries = FlagBuilder('flagkey').version(4).exclude_from_summaries(True).build() @@ -233,12 +228,13 @@ def test_identify_event_is_queued(): def test_context_is_filtered_in_identify_event(): with DefaultTestProcessor(all_attributes_private = True) as ep: + formatter = EventContextFormatter(True, []) e = EventInputIdentify(timestamp, context) ep.send_event(e) output = flush_and_get_events(ep) assert len(output) == 1 - check_identify_event(output[0], e, filtered_context) + check_identify_event(output[0], e, formatter.format_context(context)) def test_individual_feature_event_is_queued_with_index_event(): with DefaultTestProcessor() as ep: @@ -275,13 +271,14 @@ def test_exclude_can_keep_feature_event_from_summary(): def test_context_is_filtered_in_index_event(): with DefaultTestProcessor(all_attributes_private = True) as ep: + formatter = EventContextFormatter(True, []) e = EventInputEvaluation(timestamp, context, flag.key, flag, 1, 'value', None, 'default', None, True) ep.send_event(e) output = flush_and_get_events(ep) assert len(output) == 3 - check_index_event(output[0], e, filtered_context) - check_feature_event(output[1], e) + check_index_event(output[0], e, formatter.format_context(context)) + check_feature_event(output[1], e, formatter.format_context(context)) check_summary_event(output[2]) def test_two_events_for_same_context_only_produce_one_index_event(): @@ -682,7 +679,7 @@ def check_index_event(data, source: EventInput, context_json: Optional[dict] = N assert data['creationDate'] == source.timestamp assert data['context'] == (source.context.to_dict() if context_json is None else context_json) -def check_feature_event(data, source: EventInputEvaluation): +def check_feature_event(data, source: EventInputEvaluation, context_json: Optional[dict] = None): assert data['kind'] == 'feature' assert data['creationDate'] == source.timestamp assert data['key'] == source.key @@ -690,7 +687,7 @@ def check_feature_event(data, source: EventInputEvaluation): assert data.get('variation') == source.variation assert data.get('value') == source.value assert data.get('default') == source.default_value - assert data['contextKeys'] == make_context_keys(source.context) + assert data['context'] == (source.context.to_dict() if context_json is None else context_json) assert data.get('prereq_of') == None if source.prereq_of is None else source.prereq_of.key From d208612f31f5575d989983e8e9ff0d20802558af Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Tue, 16 Jan 2024 10:05:18 -0500 Subject: [PATCH 2/2] feat: Redact anonymous attributes within feature events (#246) --- contract-tests/service.py | 3 +- .../impl/events/event_context_formatter.py | 32 +++++++++++++---- ldclient/impl/events/event_processor.py | 13 ++++--- .../events/test_event_context_formatter.py | 35 +++++++++++++++++++ 4 files changed, 70 insertions(+), 13 deletions(-) diff --git a/contract-tests/service.py b/contract-tests/service.py index fea8b26..201d5b2 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -72,7 +72,8 @@ def status(): 'migrations', 'event-sampling', 'polling-gzip', - 'inline-context' + 'inline-context', + 'anonymous-redaction', ] } return (json.dumps(body), 200, {'Content-type': 'application/json'}) diff --git a/ldclient/impl/events/event_context_formatter.py b/ldclient/impl/events/event_context_formatter.py index a292953..23c3e5a 100644 --- a/ldclient/impl/events/event_context_formatter.py +++ b/ldclient/impl/events/event_context_formatter.py @@ -18,20 +18,38 @@ def __init__(self, all_attributes_private: bool, private_attributes: List[str]): self._private_attributes.append(ar) def format_context(self, context: Context) -> Dict: + """ + Formats a context for use in an analytic event, performing any + necessary attribute redaction. + """ + return self._format_context(context, False) + + def format_context_redact_anonymous(self, context: Context) -> Dict: + """ + Formats a context for use in an analytic event, performing any + necessary attribute redaction. + + If a context is anonoymous, all attributes will be redacted except for + key, kind, and anonoymous. + """ + return self._format_context(context, True) + + def _format_context(self, context: Context, redact_anonymous: bool) -> Dict: if context.multiple: out = {'kind': 'multi'} # type: Dict[str, Any] for i in range(context.individual_context_count): c = context.get_individual_context(i) if c is not None: - out[c.kind] = self._format_context_single(c, False) + out[c.kind] = self._format_context_single(c, False, redact_anonymous) return out else: - return self._format_context_single(context, True) + return self._format_context_single(context, True, redact_anonymous) - def _format_context_single(self, context: Context, include_kind: bool) -> Dict: + def _format_context_single(self, context: Context, include_kind: bool, redact_anonymous: bool) -> Dict: out = {'key': context.key} # type: Dict[str, Any] if include_kind: out['kind'] = context.kind + if context.anonymous: out['anonymous'] = True @@ -44,11 +62,11 @@ def _format_context_single(self, context: Context, include_kind: bool) -> Dict: if ar.valid: all_private.append(ar) - if context.name is not None and not self._check_whole_attr_private('name', all_private, redacted): + if context.name is not None and not self._check_whole_attr_private('name', all_private, redacted, context.anonymous and redact_anonymous): out['name'] = context.name for attr in context.custom_attributes: - if not self._check_whole_attr_private(attr, all_private, redacted): + if not self._check_whole_attr_private(attr, all_private, redacted, context.anonymous and redact_anonymous): value = context.get(attr) out[attr] = self._redact_json_value(None, attr, value, all_private, redacted) @@ -57,8 +75,8 @@ def _format_context_single(self, context: Context, include_kind: bool) -> Dict: return out - def _check_whole_attr_private(self, attr: str, all_private: List[AttributeRef], redacted: List[str]) -> bool: - if self._all_attributes_private: + def _check_whole_attr_private(self, attr: str, all_private: List[AttributeRef], redacted: List[str], redact_all: bool) -> bool: + if self._all_attributes_private or redact_all: redacted.append(attr) return True for p in all_private: diff --git a/ldclient/impl/events/event_processor.py b/ldclient/impl/events/event_processor.py index 1556238..b86b74b 100644 --- a/ldclient/impl/events/event_processor.py +++ b/ldclient/impl/events/event_processor.py @@ -65,23 +65,23 @@ def make_output_events(self, events: List[Any], summary: EventSummary): def make_output_event(self, e: Any): if isinstance(e, EventInputEvaluation): out = self._base_eval_props(e, 'feature') - out['context'] = self._process_context(e.context) + out['context'] = self._process_context(e.context, True) return out elif isinstance(e, DebugEvent): out = self._base_eval_props(e.original_input, 'debug') - out['context'] = self._process_context(e.original_input.context) + out['context'] = self._process_context(e.original_input.context, False) return out elif isinstance(e, EventInputIdentify): return { 'kind': 'identify', 'creationDate': e.timestamp, - 'context': self._process_context(e.context) + 'context': self._process_context(e.context, False) } elif isinstance(e, IndexEvent): return { 'kind': 'index', 'creationDate': e.timestamp, - 'context': self._process_context(e.context) + 'context': self._process_context(e.context, False) } elif isinstance(e, EventInputCustom): out = { @@ -193,7 +193,10 @@ def make_summary_event(self, summary: EventSummary): 'features': flags_out } - def _process_context(self, context: Context): + def _process_context(self, context: Context, redact_anonymous: bool): + if redact_anonymous: + return self._context_formatter.format_context_redact_anonymous(context) + return self._context_formatter.format_context(context) def _context_keys(self, context: Context): diff --git a/testing/impl/events/test_event_context_formatter.py b/testing/impl/events/test_event_context_formatter.py index 06662ab..6042181 100644 --- a/testing/impl/events/test_event_context_formatter.py +++ b/testing/impl/events/test_event_context_formatter.py @@ -18,6 +18,41 @@ def test_context_with_more_attributes(): 'd': 2 } +def test_context_can_redact_anonymous_attributes(): + f = EventContextFormatter(False, []) + c = Context.builder('a').name('b').anonymous(True).set('c', True).set('d', 2).build() + assert f.format_context_redact_anonymous(c) == { + 'kind': 'user', + 'key': 'a', + 'anonymous': True, + '_meta': { + 'redactedAttributes': ['name', 'c', 'd'] + } + } + +def test_multi_kind_context_can_redact_anonymous_attributes(): + f = EventContextFormatter(False, []) + user = Context.builder('user-key').name('b').anonymous(True).set('c', True).set('d', 2).build() + org = Context.builder('org-key').kind('org').name('b').set('c', True).set('d', 2).build() + multi = Context.create_multi(user, org) + + assert f.format_context_redact_anonymous(multi) == { + 'kind': 'multi', + 'user': { + 'key': 'user-key', + 'anonymous': True, + '_meta': { + 'redactedAttributes': ['name', 'c', 'd'] + } + }, + 'org': { + 'key': 'org-key', + 'name': 'b', + 'c': True, + 'd': 2 + } + } + def test_multi_context(): f = EventContextFormatter(False, []) c = Context.create_multi(