Quest/monthly use report

api/metrics/views.py

@@ -273,6 +273,8 @@ def get(self, request, *args, **kwargs):
  'user_summary': reports.UserSummaryReport,
  'spam_summary': reports.SpamSummaryReport,
  'new_user_domains': reports.NewUserDomainReport,
+ 'monthly_sessionhours': reports.MonthlySessionhoursReport,
+ 'monthly_route_use': reports.MonthlyRouteUseReport,
 }
 
 
@@ -344,7 +346,7 @@ def get(self, request, *args, report_name):
  range_field_name = 'report_yearmonth'
  range_parser = parse_yearmonth_range
  else:
- raise ValueError(f'report class must subclass DailyReport or MonthlyReport: {report_class}')
+ raise ValueError(f'VIEWABLE_REPORTS values should subclass DailyReport or MonthlyReport ("{report_name}": {report_class})')
  range_filter = range_parser(request.GET)
  search_recent = (
  report_class.search()
@@ -409,6 +411,7 @@ def _get_session_id(self, request, client_session_id=None):
  session_id_parts = [
  client_session_id,
  current_date_str,
+ now.hour,
  ]
  elif user_is_authenticated:
  session_id_parts = [

osf/metrics/counted_usage.py

@@ -3,23 +3,18 @@
 import logging
 from urllib.parse import urlsplit
 
-from elasticsearch_dsl import InnerDoc, analyzer, tokenizer
+from elasticsearch_dsl import InnerDoc
 from elasticsearch_metrics import metrics
 from elasticsearch_metrics.signals import pre_save
 from django.dispatch import receiver
 import pytz
 
-from osf.metrics.utils import stable_key
+from osf.metrics.utils import stable_key, route_prefix_analyzer
 from osf.models import Guid
 
 
 logger = logging.getLogger(__name__)
 
-route_prefix_analyzer = analyzer(
- 'route_prefix_analyzer',
- tokenizer=tokenizer('route_prefix_tokenizer', 'path_hierarchy', delimiter='.'),
-)
-
 class PageviewInfo(InnerDoc):
  """PageviewInfo
 
@@ -31,7 +26,7 @@ class PageviewInfo(InnerDoc):
  page_title = metrics.Keyword()
  route_name = metrics.Keyword(
  fields={
- 'by_prefix': metrics.Text(analyzer=route_prefix_analyzer),
+ 'by_prefix': metrics.Text(analyzer=route_prefix_analyzer, fielddata=True),
  },
  )
 

osf/metrics/reporters/__init__.py

@@ -8,6 +8,8 @@
 from .preprint_count import PreprintCountReporter
 from .user_count import UserCountReporter
 from .spam_count import SpamCountReporter
+from .monthly_sessionhours import MonthlySessionhoursReporter
+from .monthly_route_use import MonthlyRouteUseReporter
 
 
 DAILY_REPORTERS = (
@@ -24,4 +26,6 @@
 
 MONTHLY_REPORTERS = (
  SpamCountReporter,
+ MonthlySessionhoursReporter,
+ MonthlyRouteUseReporter,
 )

osf/metrics/reporters/monthly_route_use.py

@@ -0,0 +1,38 @@
+from osf.metrics.counted_usage import CountedUsage
+from osf.metrics.reports import MonthlyRouteUseReport
+from ._base import MonthlyReporter
+
+
+class MonthlyRouteUseReporter(MonthlyReporter):
+ def report(self, report_yearmonth):
+ start = report_yearmonth.as_datetime()
+ end = report_yearmonth.next().as_datetime()
+ search = (
+ CountedUsage.search()
+ .filter('range', timestamp={'gte': start, 'lte': end})
+ [:0] # just the aggregations, no hits
+ )
+ route_agg = search.aggs.bucket(
+ 'by_route',
+ 'terms',
+ field='pageview_info.route_name',
+ )
+ route_agg.metric(
+ 'total_sessions',
+ 'cardinality',
+ field='session_id',
+ precision_threshold=40000, # maximum precision
+ )
+
+ result = search.execute()
+
+ reports = []
+ for route_bucket in result.aggs.by_route.buckets:
+ report = MonthlyRouteUseReport(
+ report_yearmonth=report_yearmonth,
+ route_name=route_bucket.key,
+ use_count=route_bucket.doc_count,
+ sessionhour_count=route_bucket.total_sessions.value,
+ )
+ reports.append(report)
+ return reports

osf/metrics/reporters/monthly_sessionhours.py

@@ -0,0 +1,31 @@
+from osf.metrics.counted_usage import CountedUsage
+from osf.metrics.reports import MonthlySessionhoursReport
+from ._base import MonthlyReporter
+
+
+class MonthlySessionhoursReporter(MonthlyReporter):
+ def report(self, report_yearmonth):
+ start = report_yearmonth.as_datetime()
+ end = report_yearmonth.next().as_datetime()
+ search = (
+ CountedUsage.search()
+ .filter('range', timestamp={'gte': start, 'lte': end})
+ [:0] # just the aggregations, no hits
+ )
+ search.aggs.metric(
+ 'total_sessionhour_count',
+ 'cardinality',
+ field='session_id',
+ precision_threshold=40000, # maximum precision
+ )
+ result = search.execute()
+ total_sessionhour_count = result.aggs.total_sessionhour_count.value
+ month_timedelta = (end - start)
+ month_hours = (24 * month_timedelta.days) + int(month_timedelta.seconds / (60 * 60))
+ average_sessions_per_hour = total_sessionhour_count / month_hours
+ report = MonthlySessionhoursReport(
+ report_yearmonth=report_yearmonth,
+ total_sessionhour_count=total_sessionhour_count,
+ average_sessions_per_hour=average_sessions_per_hour,
+ )
+ return [report]

osf/metrics/reporters/spam_count.py

@@ -9,8 +9,8 @@
 class SpamCountReporter(MonthlyReporter):
 
  def report(self, report_yearmonth):
- target_month = report_yearmonth.target_month()
- next_month = report_yearmonth.next_month()
+ target_month = report_yearmonth.as_datetime()
+ next_month = report_yearmonth.next().as_datetime()
 
  report = SpamSummaryReport(
  report_yearmonth=str(report_yearmonth),

osf/metrics/reports.py

@@ -5,7 +5,7 @@
 from elasticsearch_metrics import metrics
 from elasticsearch_metrics.signals import pre_save as metrics_pre_save
 
-from osf.metrics.utils import stable_key, YearMonth
+from osf.metrics.utils import stable_key, YearMonth, route_prefix_analyzer
 
 
 class ReportInvalid(Exception):
@@ -20,7 +20,7 @@ class DailyReport(metrics.Metric):
  There's something we'd like to know about every so often,
  so let's regularly run a report and stash the results here.
  """
- DAILY_UNIQUE_FIELD = None # set in subclasses that expect multiple reports per day
+ UNIQUE_TOGETHER = ('report_date',) # override in subclasses that expect multiple reports per day
 
  report_date = metrics.Date(format='strict_date', required=True)
 
@@ -58,6 +58,7 @@ def serialize(self, data):
 class MonthlyReport(metrics.Metric):
  """MonthlyReport (abstract base for report-based metrics that run monthly)
  """
+ UNIQUE_TOGETHER = ('report_yearmonth',) # override in subclasses that expect multiple reports per month
 
  report_yearmonth = YearmonthField()
 
@@ -74,18 +75,19 @@ def set_report_id(sender, instance, **kwargs):
  # "ON CONFLICT UPDATE" behavior -- if the document
  # already exists, it will be updated rather than duplicated.
  # Cannot detect/avoid conflicts this way, but that's ok.
-
- if issubclass(sender, DailyReport):
- duf_name = instance.DAILY_UNIQUE_FIELD
- if duf_name is None:
- instance.meta.id = stable_key(instance.report_date)
- else:
- duf_value = getattr(instance, duf_name)
- if not duf_value or not isinstance(duf_value, str):
- raise ReportInvalid(f'{sender.__name__}.{duf_name} MUST have a non-empty string value (got {duf_value})')
- instance.meta.id = stable_key(instance.report_date, duf_value)
- elif issubclass(sender, MonthlyReport):
- instance.meta.id = stable_key(instance.report_yearmonth)
+ if issubclass(sender, (DailyReport, MonthlyReport)):
+ unique_together_fields = getattr(sender, 'UNIQUE_TOGETHER', None)
+ if not unique_together_fields:
+ raise ValueError(f'{sender.__name__}.UNIQUE_TOGETHER must be non-empty!')
+ unique_together_values = []
+ for field_name in unique_together_fields:
+ field_value = getattr(instance, field_name)
+ field_value_str = str(field_value)
+ if (field_value is None) or (not field_value_str):
+ raise ReportInvalid(f'{sender.__name__}.{field_name} must have a non-empty stringable value (got {field_value})')
+ unique_together_values.append(field_value_str)
+ assert len(unique_together_values) > 0
+ instance.meta.id = stable_key(*unique_together_values)
 
 
 #### BEGIN reusable inner objects #####
@@ -157,7 +159,7 @@ class DownloadCountReport(DailyReport):
 
 
 class InstitutionSummaryReport(DailyReport):
- DAILY_UNIQUE_FIELD = 'institution_id'
+ UNIQUE_TOGETHER = ('report_date', 'institution_id',)
 
  institution_id = metrics.Keyword()
  institution_name = metrics.Keyword()
@@ -169,7 +171,7 @@ class InstitutionSummaryReport(DailyReport):
 
 
 class NewUserDomainReport(DailyReport):
- DAILY_UNIQUE_FIELD = 'domain_name'
+ UNIQUE_TOGETHER = ('report_date', 'domain_name',)
 
  domain_name = metrics.Keyword()
  new_user_count = metrics.Integer()
@@ -187,7 +189,7 @@ class OsfstorageFileCountReport(DailyReport):
 
 
 class PreprintSummaryReport(DailyReport):
- DAILY_UNIQUE_FIELD = 'provider_key'
+ UNIQUE_TOGETHER = ('report_date', 'provider_key',)
 
  provider_key = metrics.Keyword()
  preprint_count = metrics.Integer()
@@ -212,5 +214,22 @@ class SpamSummaryReport(MonthlyReport):
  preprint_confirmed_spam = metrics.Integer()
  preprint_confirmed_ham = metrics.Integer()
  preprint_flagged = metrics.Integer()
- user_marked_as_spam = metrics.Integer()
+ users_marked_as_spam = metrics.Integer()
  user_marked_as_ham = metrics.Integer()
+
+
+class MonthlySessionhoursReport(MonthlyReport):
+ total_sessionhour_count = metrics.Integer()
+ average_sessions_per_hour = metrics.Float()
+
+
+class MonthlyRouteUseReport(MonthlyReport):
+ UNIQUE_TOGETHER = ('report_yearmonth', 'route_name',)
+ route_name = metrics.Keyword(
+ fields={
+ # "route_name.by_prefix" subfield for aggregating subroutes
+ 'by_prefix': metrics.Text(analyzer=route_prefix_analyzer, fielddata=True),
+ },
+ )
+ use_count = metrics.Integer()
+ sessionhour_count = metrics.Integer()

osf/metrics/utils.py

@@ -3,7 +3,7 @@
 import typing
 from hashlib import sha256
 
-import pytz
+from elasticsearch_dsl import analyzer, tokenizer
 
 
 def stable_key(*key_parts):
@@ -21,8 +21,8 @@ def stable_key(*key_parts):
 
 
 class YearMonth(typing.NamedTuple):
- year: int
- month: int
+ year: int # assumed >= 1000, < 10000
+ month: int # assumed >= 1, <= 12
 
  YEARMONTH_RE = re.compile(r'(?P<year>\d{4})-(?P<month>\d{2})')
 
@@ -45,10 +45,22 @@ def from_str(cls, input_str):
  def __str__(self):
  return f'{self.year}-{self.month:0>2}'
 
- def target_month(self):
- return datetime.datetime(self.year, self.month, 1, tzinfo=pytz.utc)
+ def as_datetime(self) -> datetime.datetime:
+ return datetime.datetime(self.year, self.month, 1, tzinfo=datetime.timezone.utc)
 
- def next_month(self):
+ def next(self):
  if self.month == 12:
- return datetime.datetime(self.year + 1, 1, 1, tzinfo=pytz.utc)
- return datetime.datetime(self.year, self.month + 1, 1, tzinfo=pytz.utc)
+ return YearMonth(self.year + 1, 1)
+ return YearMonth(self.year, self.month + 1)
+
+ def prior(self):
+ if self.month == 1:
+ return YearMonth(self.year - 1, 12)
+ return YearMonth(self.year, self.month - 1)
+
+
+# for elasticsearch fields that hold dot-delimited paths,
+# to allow querying/aggregating by prefix (e.g. 'root.to.leaf'
+# yields tokens ['root', 'root.to', 'root.to.leaf'])
+route_prefix_tokenizer = tokenizer('route_prefix_tokenizer', 'path_hierarchy', delimiter='.')
+route_prefix_analyzer = analyzer('route_prefix_analyzer', tokenizer=route_prefix_tokenizer)

osf_tests/metrics/test_daily_report.py

@@ -37,27 +37,28 @@ class Meta:
  assert report.meta.id == expected_key
  mock_save.reset_mock()
 
- def test_with_duf(self, mock_save):
+ def test_with_unique_together(self, mock_save):
  # multiple reports of this type per day, unique by given field
  class UniqueByDateAndField(DailyReport):
- DAILY_UNIQUE_FIELD = 'duf'
- duf = metrics.Keyword()
+ UNIQUE_TOGETHER = ('report_date', 'my_uniq_field',)
+
+ my_uniq_field = metrics.Keyword()
 
  class Meta:
  app_label = 'osf'
 
  today = date(2022, 5, 18)
 
  expected_blah = 'dca57e6cde89b19274ea24bc713971dab137a896b8e06d43a11a3f437cd1d151'
- blah_report = UniqueByDateAndField(report_date=today, duf='blah')
+ blah_report = UniqueByDateAndField(report_date=today, my_uniq_field='blah')
  blah_report.save()
  assert mock_save.call_count == 1
  assert mock_save.call_args[0][0] is blah_report
  assert blah_report.meta.id == expected_blah
  mock_save.reset_mock()
 
  expected_fleh = 'e7dd5ff6b087807efcfa958077dc713878f21c65af79b3ccdb5dc2409bf5ad99'
- fleh_report = UniqueByDateAndField(report_date=today, duf='fleh')
+ fleh_report = UniqueByDateAndField(report_date=today, my_uniq_field='fleh')
  fleh_report.save()
  assert mock_save.call_count == 1
  assert mock_save.call_args[0][0] is fleh_report