From 8326675c2571606412b967666419afdc490c1a39 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Thu, 22 Feb 2024 11:23:56 -0800 Subject: [PATCH] feat(beacon): Add cpu/ram usage to beacon (#65200) This adds CPU/RAM usage to the beacon. It also adds `psutil` as a dependency --------- Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com> --- requirements-base.txt | 1 + requirements-dev-frozen.txt | 2 +- requirements-dev.txt | 1 - requirements-frozen.txt | 1 + src/sentry/options/defaults.py | 5 ++ src/sentry/tasks/beacon.py | 16 ++++ src/sentry/utils/settings.py | 8 ++ src/sentry/web/client_config.py | 4 +- tests/sentry/tasks/test_beacon.py | 138 ++++++++++++++++++++++++++++-- 9 files changed, 167 insertions(+), 9 deletions(-) diff --git a/requirements-base.txt b/requirements-base.txt index b0a9f28e64f50..728becb254e5d 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -42,6 +42,7 @@ phonenumberslite>=8.12.32 Pillow>=10.2.0 progressbar2>=3.41.0 python-rapidjson>=1.4 +psutil>=5.9.2 psycopg2-binary>=2.9.9 PyJWT>=2.4.0 pydantic>=1.10.9 diff --git a/requirements-dev-frozen.txt b/requirements-dev-frozen.txt index a614608b77c04..269a5312452bc 100644 --- a/requirements-dev-frozen.txt +++ b/requirements-dev-frozen.txt @@ -127,7 +127,7 @@ progressbar2==3.41.0 prompt-toolkit==3.0.41 proto-plus==1.23.0 protobuf==4.25.2 -psutil==5.9.2 +psutil==5.9.7 psycopg2-binary==2.9.9 pyasn1==0.4.5 pyasn1-modules==0.2.4 diff --git a/requirements-dev.txt b/requirements-dev.txt index 7c16709a1235e..c6339a4e093c4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,6 @@ docker>=6 time-machine>=2.13.0 honcho>=1.1.0 openapi-core>=0.18.2 -psutil pytest>=8 pytest-cov>=4.0.0 pytest-django>=4.8.0 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 7ca4527fb904a..6f663632291c3 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -86,6 +86,7 @@ progressbar2==3.41.0 prompt-toolkit==3.0.41 proto-plus==1.23.0 protobuf==4.25.2 +psutil==5.9.7 psycopg2-binary==2.9.9 pyasn1==0.4.5 pyasn1-modules==0.2.4 diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index b1868a9ef9ba3..f32037e19ddf0 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -285,6 +285,11 @@ # Beacon register("beacon.anonymous", type=Bool, flags=FLAG_REQUIRED) +register( + "beacon.record_cpu_ram_usage", + type=Bool, + flags=FLAG_ALLOW_EMPTY | FLAG_REQUIRED, +) # Filestore (default) register("filestore.backend", default="filesystem", flags=FLAG_NOSTORE) diff --git a/src/sentry/tasks/beacon.py b/src/sentry/tasks/beacon.py index 4c896ca43c802..5b809920c9cba 100644 --- a/src/sentry/tasks/beacon.py +++ b/src/sentry/tasks/beacon.py @@ -4,6 +4,7 @@ from hashlib import sha1 from uuid import uuid4 +import psutil from django.conf import settings from django.utils import timezone @@ -120,7 +121,14 @@ def send_beacon(): # we need this to be explicitly configured and it defaults to None, # which is the same as False anonymous = options.get("beacon.anonymous") is not False + # getting an option sets it to the default value, so let's avoid doing that if for some reason consent prompt is somehow skipped because of this + send_cpu_ram_usage = ( + options.get("beacon.record_cpu_ram_usage") + if options.isset("beacon.record_cpu_ram_usage") + else False + ) event_categories_count = get_category_event_count_24h() + byte_in_gibibyte = 1024**3 payload = { "install_id": install_id, @@ -138,6 +146,14 @@ def send_beacon(): "replays.24h": event_categories_count["replay"], "profiles.24h": event_categories_count["profile"], "monitors.24h": event_categories_count["monitor"], + "cpu_cores_available": psutil.cpu_count() if send_cpu_ram_usage else None, + "cpu_percentage_utilized": psutil.cpu_percent() if send_cpu_ram_usage else None, + "ram_available_gb": ( + psutil.virtual_memory().total / byte_in_gibibyte if send_cpu_ram_usage else None + ), + "ram_percentage_utilized": ( + psutil.virtual_memory().percent if send_cpu_ram_usage else None + ), }, "packages": get_all_package_versions(), "anonymous": anonymous, diff --git a/src/sentry/utils/settings.py b/src/sentry/utils/settings.py index 632fc02c5b6f5..eceec28e4262c 100644 --- a/src/sentry/utils/settings.py +++ b/src/sentry/utils/settings.py @@ -3,3 +3,11 @@ def is_self_hosted() -> bool: from django.conf import settings return settings.SENTRY_SELF_HOSTED + + +def should_show_beacon_consent_prompt() -> bool: + from django.conf import settings + + from sentry import options + + return settings.SENTRY_SELF_HOSTED and not options.isset("beacon.record_cpu_ram_usage") diff --git a/src/sentry/web/client_config.py b/src/sentry/web/client_config.py index ca919cff94840..42b0c7e97c75f 100644 --- a/src/sentry/web/client_config.py +++ b/src/sentry/web/client_config.py @@ -37,7 +37,7 @@ from sentry.utils.assets import get_frontend_dist_prefix from sentry.utils.email import is_smtp_enabled from sentry.utils.http import is_using_customer_domain -from sentry.utils.settings import is_self_hosted +from sentry.utils.settings import is_self_hosted, should_show_beacon_consent_prompt from sentry.utils.support import get_support_mail @@ -361,6 +361,8 @@ def get_context(self) -> Mapping[str, Any]: # Maintain isOnPremise key for backcompat (plugins?). "isOnPremise": is_self_hosted(), "isSelfHosted": is_self_hosted(), + "shouldShowBeaconConsentPrompt": not self.needs_upgrade + and should_show_beacon_consent_prompt(), "invitesEnabled": settings.SENTRY_ENABLE_INVITES, "gravatarBaseUrl": settings.SENTRY_GRAVATAR_BASE_URL, "termsUrl": settings.TERMS_URL, diff --git a/tests/sentry/tasks/test_beacon.py b/tests/sentry/tasks/test_beacon.py index 1b8352a5abdf7..1ee1aa80bbe5e 100644 --- a/tests/sentry/tasks/test_beacon.py +++ b/tests/sentry/tasks/test_beacon.py @@ -1,5 +1,6 @@ import platform from datetime import timedelta +from types import SimpleNamespace from unittest.mock import patch from uuid import uuid4 @@ -18,6 +19,15 @@ @no_silo_test +@patch("psutil.cpu_count", return_value=8) +@patch("psutil.cpu_percent", return_value=50) +@patch( + "psutil.virtual_memory", + return_value=SimpleNamespace( + total=34359738368, + percent=50, + ), +) class SendBeaconTest(OutcomesSnubaTest): def setUp(self): super().setUp() @@ -87,7 +97,15 @@ def setUp(self): @patch("sentry.tasks.beacon.safe_urlopen") @patch("sentry.tasks.beacon.safe_urlread") @responses.activate - def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions): + def test_simple( + self, + safe_urlread, + safe_urlopen, + mock_get_all_package_versions, + mock_cpu_count, + mock_cpu_percent, + mock_virtual_memory, + ): self.organization self.project self.team @@ -96,6 +114,7 @@ def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions) assert options.set("system.admin-email", "foo@example.com") assert options.set("beacon.anonymous", False) + assert options.set("beacon.record_cpu_ram_usage", True) send_beacon() install_id = options.get("sentry:install-id") @@ -119,6 +138,10 @@ def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions) "replays.24h": 1, "profiles.24h": 3, "monitors.24h": 0, + "cpu_cores_available": 8, + "cpu_percentage_utilized": 50, + "ram_available_gb": 32, + "ram_percentage_utilized": 50, }, "anonymous": False, "admin_email": "foo@example.com", @@ -134,7 +157,75 @@ def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions) @patch("sentry.tasks.beacon.safe_urlopen") @patch("sentry.tasks.beacon.safe_urlread") @responses.activate - def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versions): + def test_no_cpu_ram_usage( + self, + safe_urlread, + safe_urlopen, + mock_get_all_package_versions, + mock_cpu_count, + mock_cpu_percent, + mock_virtual_memory, + ): + self.organization + self.project + self.team + mock_get_all_package_versions.return_value = {"foo": "1.0"} + safe_urlread.return_value = json.dumps({"notices": [], "version": {"stable": "1.0.0"}}) + + assert options.set("system.admin-email", "foo@example.com") + assert options.set("beacon.anonymous", False) + assert options.set("beacon.record_cpu_ram_usage", False) + send_beacon() + + install_id = options.get("sentry:install-id") + assert install_id and len(install_id) == 40 + + safe_urlopen.assert_called_once_with( + BEACON_URL, + json={ + "install_id": install_id, + "version": sentry.get_version(), + "docker": sentry.is_docker(), + "python_version": platform.python_version(), + "data": { + "organizations": 2, + "users": 1, + "projects": 2, + "teams": 2, + "events.24h": 8, # We expect the number of events to be the sum of events from two orgs. First org has 5 events while the second org has 3 events. + "errors.24h": 8, + "transactions.24h": 2, + "replays.24h": 1, + "profiles.24h": 3, + "monitors.24h": 0, + "cpu_cores_available": None, + "cpu_percentage_utilized": None, + "ram_available_gb": None, + "ram_percentage_utilized": None, + }, + "anonymous": False, + "admin_email": "foo@example.com", + "packages": mock_get_all_package_versions.return_value, + }, + timeout=5, + ) + safe_urlread.assert_called_once_with(safe_urlopen.return_value) + + assert options.get("sentry:latest_version") == "1.0.0" + + @patch("sentry.tasks.beacon.get_all_package_versions") + @patch("sentry.tasks.beacon.safe_urlopen") + @patch("sentry.tasks.beacon.safe_urlread") + @responses.activate + def test_anonymous( + self, + safe_urlread, + safe_urlopen, + mock_get_all_package_versions, + mock_cpu_count, + mock_cpu_percent, + mock_virtual_memory, + ): self.organization self.project self.team @@ -143,6 +234,7 @@ def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versio assert options.set("system.admin-email", "foo@example.com") assert options.set("beacon.anonymous", True) + assert options.set("beacon.record_cpu_ram_usage", True) send_beacon() install_id = options.get("sentry:install-id") @@ -166,6 +258,10 @@ def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versio "replays.24h": 1, "profiles.24h": 3, "monitors.24h": 0, + "cpu_cores_available": 8, + "cpu_percentage_utilized": 50, + "ram_available_gb": 32, + "ram_percentage_utilized": 50, }, "anonymous": True, "packages": mock_get_all_package_versions.return_value, @@ -180,7 +276,15 @@ def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versio @patch("sentry.tasks.beacon.safe_urlopen") @patch("sentry.tasks.beacon.safe_urlread") @responses.activate - def test_with_broadcasts(self, safe_urlread, safe_urlopen, mock_get_all_package_versions): + def test_with_broadcasts( + self, + safe_urlread, + safe_urlopen, + mock_get_all_package_versions, + mock_cpu_count, + mock_cpu_percent, + mock_virtual_memory, + ): broadcast_id = uuid4().hex mock_get_all_package_versions.return_value = {} safe_urlread.return_value = json.dumps( @@ -236,7 +340,15 @@ def test_with_broadcasts(self, safe_urlread, safe_urlopen, mock_get_all_package_ @patch("sentry.tasks.beacon.safe_urlopen") @patch("sentry.tasks.beacon.safe_urlread") @responses.activate - def test_disabled(self, safe_urlread, safe_urlopen, mock_get_all_package_versions): + def test_disabled( + self, + safe_urlread, + safe_urlopen, + mock_get_all_package_versions, + mock_cpu_count, + mock_cpu_percent, + mock_virtual_memory, + ): mock_get_all_package_versions.return_value = {"foo": "1.0"} with self.settings(SENTRY_BEACON=False): @@ -248,7 +360,15 @@ def test_disabled(self, safe_urlread, safe_urlopen, mock_get_all_package_version @patch("sentry.tasks.beacon.safe_urlopen") @patch("sentry.tasks.beacon.safe_urlread") @responses.activate - def test_debug(self, safe_urlread, safe_urlopen, mock_get_all_package_versions): + def test_debug( + self, + safe_urlread, + safe_urlopen, + mock_get_all_package_versions, + mock_cpu_count, + mock_cpu_percent, + mock_virtual_memory, + ): mock_get_all_package_versions.return_value = {"foo": "1.0"} with self.settings(DEBUG=True): @@ -258,7 +378,13 @@ def test_debug(self, safe_urlread, safe_urlopen, mock_get_all_package_versions): @patch("sentry.tasks.beacon.safe_urlopen") @responses.activate - def test_metrics(self, safe_urlopen): + def test_metrics( + self, + safe_urlopen, + mock_cpu_count, + mock_cpu_percent, + mock_virtual_memory, + ): metrics = [ { "description": "SentryApp",