From 7e4fdad5f5aa54ea968c38ad57939c08e90e7eb0 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Fri, 4 Aug 2023 15:28:35 -0700 Subject: [PATCH 01/12] Emit escalating metrics --- src/sentry/tasks/post_process.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index 748a819ae77bb..33a7f20363b3d 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -16,6 +16,8 @@ from sentry.issues.grouptype import GroupCategory from sentry.issues.issue_occurrence import IssueOccurrence from sentry.killswitches import killswitch_matches_context +from sentry.sentry_metrics.client.kafka import KafkaMetricsBackend +from sentry.sentry_metrics.use_case_id_registry import UseCaseID from sentry.signals import event_processed, issue_unignored, transaction_processed from sentry.tasks.base import instrumented_task from sentry.utils import metrics @@ -115,6 +117,23 @@ def _capture_event_stats(event: Event) -> None: metrics.timing("events.size.data", event.size, tags=tags) +def _update_escalating_metrics(event: Event) -> None: + """ + Update metrics for escalating issues when an event is processed. + """ + metrics_backend = KafkaMetricsBackend() + metrics_backend.counter( + UseCaseID.ESCALATING_ISSUES, + org_id=event.project.organization_id, + project_id=event.project.id, + metric_name="event_ingested", + value=1, + tags={"group": str(event.group_id)}, + unit=None, + ) + metrics_backend.close() + + def _capture_group_stats(job: PostProcessJob) -> None: event = job["event"] if not job["group_state"]["is_new"] or not should_write_event_stats(event): @@ -571,6 +590,7 @@ def get_event_raise_exception() -> Event: update_event_groups(event, group_states) bind_organization_context(event.project.organization) _capture_event_stats(event) + _update_escalating_metrics(event) group_events: Mapping[int, GroupEvent] = { ge.group_id: ge for ge in list(event.build_group_events()) From c7b90523ab3b5dfc52afa013b7df6b3cd561a6e6 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Fri, 4 Aug 2023 15:41:57 -0700 Subject: [PATCH 02/12] Switch to the generic_metrics_backend wrapper --- src/sentry/tasks/post_process.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index 33a7f20363b3d..54d82f99cddcf 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -16,7 +16,7 @@ from sentry.issues.grouptype import GroupCategory from sentry.issues.issue_occurrence import IssueOccurrence from sentry.killswitches import killswitch_matches_context -from sentry.sentry_metrics.client.kafka import KafkaMetricsBackend +from sentry.sentry_metrics.client import generic_metrics_backend from sentry.sentry_metrics.use_case_id_registry import UseCaseID from sentry.signals import event_processed, issue_unignored, transaction_processed from sentry.tasks.base import instrumented_task @@ -121,8 +121,7 @@ def _update_escalating_metrics(event: Event) -> None: """ Update metrics for escalating issues when an event is processed. """ - metrics_backend = KafkaMetricsBackend() - metrics_backend.counter( + generic_metrics_backend.counter( UseCaseID.ESCALATING_ISSUES, org_id=event.project.organization_id, project_id=event.project.id, @@ -131,7 +130,6 @@ def _update_escalating_metrics(event: Event) -> None: tags={"group": str(event.group_id)}, unit=None, ) - metrics_backend.close() def _capture_group_stats(job: PostProcessJob) -> None: From 08f6125cb4249ea2407dfa43735df821748d1df1 Mon Sep 17 00:00:00 2001 From: Riya Chakraborty Date: Tue, 8 Aug 2023 10:27:46 -0500 Subject: [PATCH 03/12] use default project id --- tests/sentry/sentry_metrics/test_snuba.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/sentry/sentry_metrics/test_snuba.py b/tests/sentry/sentry_metrics/test_snuba.py index 34cd96d8ddc07..3f6212551cb0f 100644 --- a/tests/sentry/sentry_metrics/test_snuba.py +++ b/tests/sentry/sentry_metrics/test_snuba.py @@ -13,7 +13,6 @@ class MetricsInterfaceTestCase(BaseMetricsLayerTestCase, TestCase, GenericMetricsTestMixIn): def setUp(self): super().setUp() - self.test_project = self.create_project() class SnubaMetricsInterfaceTest(MetricsInterfaceTestCase): @@ -33,7 +32,7 @@ def test_count_query(self): generic_metrics_backend.distribution( self.use_case_id, self.organization.id, - self.test_project.id, + self.project.id, self.metric_name, [100, 200, 300], {}, @@ -49,7 +48,6 @@ def test_count_query(self): metric_mri=self.get_mri(self.metric_name, "d", self.use_case_id, self.unit), ), ], - project_ids=[self.test_project.id], groupby=[], orderby=[], limit=Limit(limit=1), @@ -58,7 +56,7 @@ def test_count_query(self): ) data = get_series( - [self.test_project], + [self.project], metrics_query=metrics_query, include_meta=True, use_case_id=self.use_case_id, From 92267bc4a0ca3d4334a8b3c6024f3ebe7babb016 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Tue, 8 Aug 2023 19:49:08 -0700 Subject: [PATCH 04/12] Add a feature flag --- src/sentry/conf/server.py | 2 ++ src/sentry/features/__init__.py | 1 + src/sentry/tasks/post_process.py | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 69198104d736c..01f032a33db00 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -1378,6 +1378,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: "organizations:escalating-issues-msteams": False, # Enable archive/escalating issue workflow features in v2 "organizations:escalating-issues-v2": False, + # Enable emiting escalating data to the metrics backend + "organizations:escalating-metrics-backend": False, # Enable the new issue states and substates "organizations:issue-states": False, # Allows an org to have a larger set of project ownership rules per project diff --git a/src/sentry/features/__init__.py b/src/sentry/features/__init__.py index 4f6c85f6f21cb..bb85a8291c6eb 100644 --- a/src/sentry/features/__init__.py +++ b/src/sentry/features/__init__.py @@ -231,6 +231,7 @@ default_manager.add("organizations:escalating-issues", OrganizationFeature, FeatureHandlerStrategy.INTERNAL) default_manager.add("organizations:escalating-issues-msteams", OrganizationFeature, FeatureHandlerStrategy.REMOTE) default_manager.add("organizations:escalating-issues-v2", OrganizationFeature, FeatureHandlerStrategy.INTERNAL) +default_manager.add("organizations:escalating-metrics-backend", OrganizationFeature, FeatureHandlerStrategy.INTERNAL) default_manager.add("organizations:integrations-gh-invite", OrganizationFeature, FeatureHandlerStrategy.REMOTE) default_manager.add("organizations:issue-states", OrganizationFeature, FeatureHandlerStrategy.INTERNAL) default_manager.add("organizations:event-attachments", OrganizationFeature, FeatureHandlerStrategy.INTERNAL) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index 54d82f99cddcf..4dfcbf99f1ae9 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -588,7 +588,8 @@ def get_event_raise_exception() -> Event: update_event_groups(event, group_states) bind_organization_context(event.project.organization) _capture_event_stats(event) - _update_escalating_metrics(event) + if features.has("organizations:escalating-metrics-backend", event.project.organization): + _update_escalating_metrics(event) group_events: Mapping[int, GroupEvent] = { ge.group_id: ge for ge in list(event.build_group_events()) From 00132773b8ae56b85561312be38c24e6dee40e56 Mon Sep 17 00:00:00 2001 From: Riya Chakraborty Date: Thu, 10 Aug 2023 11:08:06 -0500 Subject: [PATCH 05/12] fix test_notifications --- .../notifications/test_notifications.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/sentry/notifications/test_notifications.py b/tests/sentry/notifications/test_notifications.py index 33c3e5e8feb6b..30c4d73f20ffb 100644 --- a/tests/sentry/notifications/test_notifications.py +++ b/tests/sentry/notifications/test_notifications.py @@ -42,9 +42,9 @@ def make_event(**kwargs): return result -def get_attachment(): +def get_attachment(response_num): assert len(responses.calls) >= 1 - data = parse_qs(responses.calls[0].request.body) + data = parse_qs(responses.calls[response_num].request.body) assert "text" in data assert "attachments" in data attachments = json.loads(data["attachments"][0]) @@ -131,7 +131,7 @@ def test_sends_note_notification(self): assert isinstance(msg.alternatives[0][0], str) assert "blah blah

" in msg.alternatives[0][0] - attachment, text = get_attachment() + attachment, text = get_attachment(0) # check the Slack version assert text == f"New comment by {self.name}" assert attachment["title"] == f"{self.group.title}" @@ -171,7 +171,7 @@ def test_sends_unassignment_notification(self): assert isinstance(msg.alternatives[0][0], str) assert f"{self.user.username} unassigned" in msg.alternatives[0][0] - attachment, text = get_attachment() + attachment, text = get_attachment(0) assert text == f"Issue unassigned by {self.name}" assert attachment["title"] == self.group.title @@ -201,7 +201,7 @@ def test_sends_resolution_notification(self, record_analytics): assert isinstance(msg.alternatives[0][0], str) assert f"{self.short_id} as resolved

" in msg.alternatives[0][0] - attachment, text = get_attachment() + attachment, text = get_attachment(0) assert ( text @@ -258,7 +258,7 @@ def test_sends_deployment_notification(self, record_analytics): in msg.alternatives[0][0] ) - attachment, text = get_attachment() + attachment, text = get_attachment(0) assert ( text @@ -325,7 +325,7 @@ def test_sends_regression_notification(self, record_analytics): assert isinstance(msg.alternatives[0][0], str) assert f"{group.qualified_short_id} as a regression

" in msg.alternatives[0][0] - attachment, text = get_attachment() + attachment, text = get_attachment(0) assert text == "Issue marked as regression" assert ( @@ -379,7 +379,7 @@ def test_sends_resolved_in_release_notification(self, record_analytics): f'text-decoration: none">{self.short_id} as resolved in' in msg.alternatives[0][0] ) - attachment, text = get_attachment() + attachment, text = get_attachment(0) assert text == f"Issue marked as resolved in {parsed_version} by {self.name}" assert attachment["title"] == self.group.title assert ( @@ -440,6 +440,13 @@ def test_sends_issue_notification(self, record_analytics): project_id=self.project.id, ) cache_key = write_event_to_cache(event) + responses.add( + method=responses.POST, + url="http://127.0.0.1:1218/tests/entities/generic_metrics_counters/insert", + body='{"ok": true}', + status=200, + content_type="application/json", + ) with self.tasks(): post_process_group( is_new=True, @@ -457,7 +464,7 @@ def test_sends_issue_notification(self, record_analytics): assert isinstance(msg.alternatives[0][0], str) assert "Hello world" in msg.alternatives[0][0] - attachment, text = get_attachment() + attachment, text = get_attachment(1) assert attachment["title"] == "Hello world" assert ( From b7a7bf913dfb78aeb2a9de46222a6d94f3a0a943 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Thu, 10 Aug 2023 16:41:43 -0700 Subject: [PATCH 06/12] Add a passthrough for metrics BE requests --- tests/relay_integration/lang/javascript/test_plugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/relay_integration/lang/javascript/test_plugin.py b/tests/relay_integration/lang/javascript/test_plugin.py index 6f9a0a1fe1232..7bbd8ce92009e 100644 --- a/tests/relay_integration/lang/javascript/test_plugin.py +++ b/tests/relay_integration/lang/javascript/test_plugin.py @@ -1,4 +1,5 @@ import os.path +import re import zipfile from base64 import b64encode from datetime import timedelta @@ -1308,6 +1309,7 @@ def test_no_fetch_from_http(self): body=load_fixture("node_app.min.js.map"), content_type="application/javascript; charset=utf-8", ) + responses.add_passthru(re.compile(".*/tests/entities/generic_metrics_counters/insert")) data = { "timestamp": self.min_ago, @@ -1383,6 +1385,8 @@ def test_html_file_with_query_param_ending_with_js_extension(self): "" ), ) + responses.add_passthru(re.compile(".*/tests/entities/generic_metrics_counters/insert")) + data = { "timestamp": self.min_ago, "message": "hello", From 92e47b07a065afa1c6a90f81cce2600528edca00 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Thu, 10 Aug 2023 16:50:58 -0700 Subject: [PATCH 07/12] TESTING DO NOT MERGE - enable feature flag to run CI --- src/sentry/conf/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 01f032a33db00..1f55e83a68ec3 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -1379,7 +1379,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: # Enable archive/escalating issue workflow features in v2 "organizations:escalating-issues-v2": False, # Enable emiting escalating data to the metrics backend - "organizations:escalating-metrics-backend": False, + "organizations:escalating-metrics-backend": True, # Enable the new issue states and substates "organizations:issue-states": False, # Allows an org to have a larger set of project ownership rules per project From 8667a08cb269ed7e59639c9677da04ff5c49a6d6 Mon Sep 17 00:00:00 2001 From: Riya Chakraborty Date: Thu, 10 Aug 2023 20:20:11 -0500 Subject: [PATCH 08/12] try adding responses --- .../lang/javascript/test_plugin.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/relay_integration/lang/javascript/test_plugin.py b/tests/relay_integration/lang/javascript/test_plugin.py index 7bbd8ce92009e..048c39718fe12 100644 --- a/tests/relay_integration/lang/javascript/test_plugin.py +++ b/tests/relay_integration/lang/javascript/test_plugin.py @@ -1,5 +1,4 @@ import os.path -import re import zipfile from base64 import b64encode from datetime import timedelta @@ -1309,7 +1308,13 @@ def test_no_fetch_from_http(self): body=load_fixture("node_app.min.js.map"), content_type="application/javascript; charset=utf-8", ) - responses.add_passthru(re.compile(".*/tests/entities/generic_metrics_counters/insert")) + responses.add( + method=responses.POST, + url="http://127.0.0.1:1218/tests/entities/generic_metrics_counters/insert", + body='{"ok": true}', + status=200, + content_type="application/json", + ) data = { "timestamp": self.min_ago, @@ -1385,7 +1390,13 @@ def test_html_file_with_query_param_ending_with_js_extension(self): "" ), ) - responses.add_passthru(re.compile(".*/tests/entities/generic_metrics_counters/insert")) + responses.add( + method=responses.POST, + url="http://127.0.0.1:1218/tests/entities/generic_metrics_counters/insert", + body='{"ok": true}', + status=200, + content_type="application/json", + ) data = { "timestamp": self.min_ago, From d16a6fa31e23325d79e6d25bd10d29338fea4f30 Mon Sep 17 00:00:00 2001 From: Riya Chakraborty Date: Thu, 10 Aug 2023 20:54:30 -0500 Subject: [PATCH 09/12] add feature (for now) --- tests/sentry/api/serializers/test_organization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/sentry/api/serializers/test_organization.py b/tests/sentry/api/serializers/test_organization.py index 17f55978c48a7..1e661d4ab79f1 100644 --- a/tests/sentry/api/serializers/test_organization.py +++ b/tests/sentry/api/serializers/test_organization.py @@ -90,6 +90,7 @@ def test_simple(self): "performance-issues-search", "transaction-name-normalize", "transaction-name-mark-scrubbed-as-sanitized", + "escalating-metrics-backend", } @mock.patch("sentry.features.batch_has") From d71691ad0c5371e209983583a891a95283c03b01 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Thu, 10 Aug 2023 22:34:15 -0700 Subject: [PATCH 10/12] Revert to a passthrough for the metrics request --- .../lang/javascript/test_plugin.py | 17 +++++------------ .../sentry/notifications/test_notifications.py | 11 ++++------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/tests/relay_integration/lang/javascript/test_plugin.py b/tests/relay_integration/lang/javascript/test_plugin.py index 048c39718fe12..9a19ed3129637 100644 --- a/tests/relay_integration/lang/javascript/test_plugin.py +++ b/tests/relay_integration/lang/javascript/test_plugin.py @@ -8,6 +8,7 @@ import pytest import responses +from django.conf import settings from django.core.files.base import ContentFile from django.utils import timezone @@ -1308,12 +1309,8 @@ def test_no_fetch_from_http(self): body=load_fixture("node_app.min.js.map"), content_type="application/javascript; charset=utf-8", ) - responses.add( - method=responses.POST, - url="http://127.0.0.1:1218/tests/entities/generic_metrics_counters/insert", - body='{"ok": true}', - status=200, - content_type="application/json", + responses.add_passthru( + settings.SENTRY_SNUBA + "/tests/entities/generic_metrics_counters/insert", ) data = { @@ -1390,12 +1387,8 @@ def test_html_file_with_query_param_ending_with_js_extension(self): "" ), ) - responses.add( - method=responses.POST, - url="http://127.0.0.1:1218/tests/entities/generic_metrics_counters/insert", - body='{"ok": true}', - status=200, - content_type="application/json", + responses.add_passthru( + settings.SENTRY_SNUBA + "/tests/entities/generic_metrics_counters/insert", ) data = { diff --git a/tests/sentry/notifications/test_notifications.py b/tests/sentry/notifications/test_notifications.py index 30c4d73f20ffb..cc27e031f88a0 100644 --- a/tests/sentry/notifications/test_notifications.py +++ b/tests/sentry/notifications/test_notifications.py @@ -5,6 +5,7 @@ from urllib.parse import parse_qs import responses +from django.conf import settings from django.core import mail from django.core.mail.message import EmailMultiAlternatives from django.utils import timezone @@ -106,6 +107,9 @@ def setUp(self): status=200, content_type="application/json", ) + responses.add_passthru( + settings.SENTRY_SNUBA + "/tests/entities/generic_metrics_counters/insert", + ) self.name = self.user.get_display_name() self.short_id = self.group.qualified_short_id @@ -440,13 +444,6 @@ def test_sends_issue_notification(self, record_analytics): project_id=self.project.id, ) cache_key = write_event_to_cache(event) - responses.add( - method=responses.POST, - url="http://127.0.0.1:1218/tests/entities/generic_metrics_counters/insert", - body='{"ok": true}', - status=200, - content_type="application/json", - ) with self.tasks(): post_process_group( is_new=True, From 144e44c6bdc3ffac19b28b12ae2da5929c477075 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Fri, 11 Aug 2023 11:27:46 -0700 Subject: [PATCH 11/12] Revert feature flag to False --- src/sentry/conf/server.py | 2 +- tests/sentry/api/serializers/test_organization.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 588f6c34bf6c0..ec00270c9a535 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -1379,7 +1379,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: # Enable archive/escalating issue workflow features in v2 "organizations:escalating-issues-v2": False, # Enable emiting escalating data to the metrics backend - "organizations:escalating-metrics-backend": True, + "organizations:escalating-metrics-backend": False, # Allows an org to have a larger set of project ownership rules per project "organizations:higher-ownership-limit": False, # Enable Monitors (Crons) view diff --git a/tests/sentry/api/serializers/test_organization.py b/tests/sentry/api/serializers/test_organization.py index 1e661d4ab79f1..17f55978c48a7 100644 --- a/tests/sentry/api/serializers/test_organization.py +++ b/tests/sentry/api/serializers/test_organization.py @@ -90,7 +90,6 @@ def test_simple(self): "performance-issues-search", "transaction-name-normalize", "transaction-name-mark-scrubbed-as-sanitized", - "escalating-metrics-backend", } @mock.patch("sentry.features.batch_has") From 9aaffb5a19452a7a24da5d96f7a6a34d33f71307 Mon Sep 17 00:00:00 2001 From: Snigdha Sharma Date: Mon, 14 Aug 2023 09:20:18 -0700 Subject: [PATCH 12/12] Revert changes to get_attachment --- .../sentry/notifications/test_notifications.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/sentry/notifications/test_notifications.py b/tests/sentry/notifications/test_notifications.py index cc27e031f88a0..574ec002f6137 100644 --- a/tests/sentry/notifications/test_notifications.py +++ b/tests/sentry/notifications/test_notifications.py @@ -43,9 +43,9 @@ def make_event(**kwargs): return result -def get_attachment(response_num): +def get_attachment(): assert len(responses.calls) >= 1 - data = parse_qs(responses.calls[response_num].request.body) + data = parse_qs(responses.calls[0].request.body) assert "text" in data assert "attachments" in data attachments = json.loads(data["attachments"][0]) @@ -135,7 +135,7 @@ def test_sends_note_notification(self): assert isinstance(msg.alternatives[0][0], str) assert "blah blah

" in msg.alternatives[0][0] - attachment, text = get_attachment(0) + attachment, text = get_attachment() # check the Slack version assert text == f"New comment by {self.name}" assert attachment["title"] == f"{self.group.title}" @@ -175,7 +175,7 @@ def test_sends_unassignment_notification(self): assert isinstance(msg.alternatives[0][0], str) assert f"{self.user.username} unassigned" in msg.alternatives[0][0] - attachment, text = get_attachment(0) + attachment, text = get_attachment() assert text == f"Issue unassigned by {self.name}" assert attachment["title"] == self.group.title @@ -205,7 +205,7 @@ def test_sends_resolution_notification(self, record_analytics): assert isinstance(msg.alternatives[0][0], str) assert f"{self.short_id} as resolved

" in msg.alternatives[0][0] - attachment, text = get_attachment(0) + attachment, text = get_attachment() assert ( text @@ -262,7 +262,7 @@ def test_sends_deployment_notification(self, record_analytics): in msg.alternatives[0][0] ) - attachment, text = get_attachment(0) + attachment, text = get_attachment() assert ( text @@ -329,7 +329,7 @@ def test_sends_regression_notification(self, record_analytics): assert isinstance(msg.alternatives[0][0], str) assert f"{group.qualified_short_id} as a regression

" in msg.alternatives[0][0] - attachment, text = get_attachment(0) + attachment, text = get_attachment() assert text == "Issue marked as regression" assert ( @@ -383,7 +383,7 @@ def test_sends_resolved_in_release_notification(self, record_analytics): f'text-decoration: none">{self.short_id} as resolved in' in msg.alternatives[0][0] ) - attachment, text = get_attachment(0) + attachment, text = get_attachment() assert text == f"Issue marked as resolved in {parsed_version} by {self.name}" assert attachment["title"] == self.group.title assert ( @@ -461,7 +461,7 @@ def test_sends_issue_notification(self, record_analytics): assert isinstance(msg.alternatives[0][0], str) assert "Hello world" in msg.alternatives[0][0] - attachment, text = get_attachment(1) + attachment, text = get_attachment() assert attachment["title"] == "Hello world" assert (