Create MITx Online user activities intermediate model, updates to xPro (

#795)

* Create MITx Online user activities intermediate model, updates to xPro

* updates for production data

* Create MITx Online user activities intermediate model, updates to xPro

src/ol_dbt/models/intermediate/mitxonline/_int_mitxonline__models.yml

@@ -1280,3 +1280,147 @@ models:
   tests:
   - dbt_expectations.expect_table_row_count_to_equal_other_table:
       compare_model: ref('stg__mitxonline__app__postgres__users_user')
+
+- name: int__mitxonline__user_courseactivity_video
+  description: It tracks MITx Online open edX users video activities within a course
+  columns:
+  - name: user_username
+    description: str, username of the open edX user who caused the event to be emitted.
+    tests:
+    - not_null
+  - name: openedx_user_id
+    description: int, reference user id in auth_user from open edX.
+    tests:
+    - not_null
+  - name: courserun_readable_id
+    description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}
+    tests:
+    - not_null
+  - name: useractivity_event_source
+    description: str, specifies the source of the interaction that triggered the event.
+      The values are either browser or mobile for this event
+    tests:
+    - not_null
+  - name: useractivity_event_type
+    description: str, type of video event triggered - play_video - When a user selects
+      the video player’s play control pause_video - When a user selects the video
+      player’s pause control stop_video - When the video player reaches the end of
+      the video file and play automatically stops
+    tests:
+    - not_null
+  - name: useractivity_video_id
+    description: str, hash code for the video being watched. This value is part of
+      the courseware_studentmodule.module_id
+    tests:
+    - not_null
+  - name: useractivity_video_duration
+    description: number, The length of the video file, in seconds.
+    tests:
+    - not_null
+  - name: useractivity_video_currenttime
+    description: number, The time in the video when this event was emitted
+    tests:
+    - not_null
+  - name: useractivity_timestamp
+    description: timestamp, time when this event was emitted
+    tests:
+    - not_null
+  tests:
+  - dbt_expectations.expect_compound_columns_to_be_unique:
+      column_list: ["user_username", "courserun_readable_id", "useractivity_event_type",
+        "useractivity_video_id", "useractivity_video_currenttime", "useractivity_timestamp"]
+
+- name: int__mitxonline__user_courseactivity_problemcheck
+  description: It tracks MITx Online open edX users problem_check activities within
+    a course
+  columns:
+  - name: user_username
+    description: str, username of the open edX user who caused the event to be emitted.
+    tests:
+    - not_null
+  - name: openedx_user_id
+    description: int, reference user id in auth_user from open edX.
+    tests:
+    - not_null
+  - name: courserun_readable_id
+    description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}
+    tests:
+    - not_null
+  - name: useractivity_event_type
+    description: str, problem_check - when a problem is successfully checked.
+    tests:
+    - not_null
+  - name: useractivity_problem_id
+    description: str, Unique ID for this problem in a course. It's recorded as a URL
+      format - block-v1:{org)+{course ID}+type@problem+block@{hash code}
+    tests:
+    - not_null
+  - name: useractivity_problem_name
+    description: str, display name of this problem in a course
+    tests:
+    - not_null
+  - name: useractivity_problem_attempts
+    description: number, The number of times the user attempted to answer this problem
+    tests:
+    - not_null
+  - name: useractivity_problem_success
+    description: str, It's either 'correct' or 'incorrect'
+    tests:
+    - not_null
+  - name: useractivity_problem_current_grade
+    description: number, current grade value for this user
+    tests:
+    - not_null
+  - name: useractivity_problem_max_grade
+    description: number, Maximum possible grade value for this problem
+    tests:
+    - not_null
+  - name: useractivity_timestamp
+    description: timestamp, time when this event was emitted
+    tests:
+    - not_null
+  tests:
+  - dbt_expectations.expect_compound_columns_to_be_unique:
+      column_list: ["user_username", "courserun_readable_id", "useractivity_problem_id",
+        "useractivity_timestamp"]
+
+- name: int__mitxonline__user_courseactivities
+  description: MITx Online open edX user activities aggregated statistics per course
+    from tracking logs
+  columns:
+  - name: user_username
+    description: str, username of the open edX user who caused the event to be emitted.
+    tests:
+    - not_null
+  - name: courserun_readable_id
+    description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}.
+    tests:
+    - not_null
+  - name: courseactivity_num_events
+    description: int, number of tracking log events
+    tests:
+    - not_null
+  - name: courseactivity_num_days_activity
+    description: int, number of days with activity
+    tests:
+    - not_null
+  - name: courseactivity_first_event_timestamp
+    description: timestamp, timestamp of user's first event within a course
+    tests:
+    - not_null
+  - name: courseactivity_last_event_timestamp
+    description: timestamp, timestamp of user's last event within a course
+    tests:
+    - not_null
+  - name: courseactivity_num_play_video
+    description: int, number of play_video events
+  - name: courseactivity_num_unique_play_video
+    description: int, number of unique videos played within a course
+  - name: courseactivity_last_play_video_timestamp
+    description: timestamp, timestamp of user's last play_video event within a course
+  - name: courseactivity_last_problem_check_timestamp
+    description: timestamp, timestamp of user's last problem_check event within a
+      course
+  tests:
+  - dbt_expectations.expect_compound_columns_to_be_unique:
+      column_list: ["user_username", "courserun_readable_id"]

src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivities.sql

@@ -0,0 +1,65 @@
+with course_activities as (
+    select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }}
+    where courserun_readable_id is not null
+)
+
+, course_activities_video as (
+    select * from {{ ref('int__mitxonline__user_courseactivity_video') }}
+)
+
+, problem_check as (
+    select * from {{ ref('int__mitxonline__user_courseactivity_problemcheck') }}
+)
+
+, problem_check_stats as (
+    select
+        user_username
+        , courserun_readable_id
+        , max(useractivity_timestamp) as courseactivity_last_problem_check_timestamp
+    from problem_check
+    group by user_username, courserun_readable_id
+)
+
+, play_video_stats as (
+    select
+        user_username
+        , courserun_readable_id
+        , count(distinct useractivity_video_id) as courseactivity_num_unique_play_video
+        , count(*) as courseactivity_num_play_video
+        , max(useractivity_timestamp) as courseactivity_last_play_video_timestamp
+    from course_activities_video
+    where useractivity_event_type = 'play_video'
+    group by user_username, courserun_readable_id
+)
+
+, all_course_activities_stats as (
+    select
+        user_username
+        , courserun_readable_id
+        , count(distinct date(from_iso8601_timestamp(useractivity_timestamp))) as courseactivity_num_days_activity
+        , count(*) as courseactivity_num_events
+        , min(useractivity_timestamp) as courseactivity_first_event_timestamp
+        , max(useractivity_timestamp) as courseactivity_last_event_timestamp
+    from course_activities
+    group by user_username, courserun_readable_id
+)
+
+select
+    all_course_activities_stats.user_username
+    , all_course_activities_stats.courserun_readable_id
+    , all_course_activities_stats.courseactivity_num_days_activity
+    , all_course_activities_stats.courseactivity_num_events
+    , play_video_stats.courseactivity_num_unique_play_video
+    , play_video_stats.courseactivity_num_play_video
+    , play_video_stats.courseactivity_last_play_video_timestamp
+    , problem_check_stats.courseactivity_last_problem_check_timestamp
+    , all_course_activities_stats.courseactivity_first_event_timestamp
+    , all_course_activities_stats.courseactivity_last_event_timestamp
+from all_course_activities_stats
+left join play_video_stats
+    on
+        all_course_activities_stats.user_username = play_video_stats.user_username
+        and all_course_activities_stats.courserun_readable_id = play_video_stats.courserun_readable_id
+left join problem_check_stats on
+    all_course_activities_stats.user_username = problem_check_stats.user_username
+    and all_course_activities_stats.courserun_readable_id = problem_check_stats.courserun_readable_id

src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_problemcheck.sql

@@ -0,0 +1,24 @@
+{{ config(materialized='view') }}
+
+with course_activities as (
+    select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }}
+    where courserun_readable_id is not null
+)
+
+select
+    user_username
+    , courserun_readable_id
+    , openedx_user_id
+    , useractivity_event_type
+    , useractivity_timestamp
+    , json_query(useractivity_context_object, 'lax $.module.display_name' omit quotes) as useractivity_problem_name
+    , json_query(useractivity_event_object, 'lax $.problem_id' omit quotes) as useractivity_problem_id
+    , json_query(useractivity_event_object, 'lax $.attempts' omit quotes) as useractivity_problem_attempts
+    , json_query(useractivity_event_object, 'lax $.success' omit quotes) as useractivity_problem_success
+    , json_query(useractivity_event_object, 'lax $.grade' omit quotes) as useractivity_problem_current_grade
+    , json_query(useractivity_event_object, 'lax $.max_grade' omit quotes) as useractivity_problem_max_grade
+from course_activities
+where useractivity_event_type = 'problem_check'
+--- This event emitted by the browser contain all of the GET parameters,
+--  only events emitted by the server are useful
+and useractivity_event_source = 'server'

src/ol_dbt/models/intermediate/mitxpro/subqueries/__mitxpro__user_courseactivity_video.sql → src/ol_dbt/models/intermediate/mitxonline/int__mitxonline__user_courseactivity_video.sql

@@ -1,6 +1,8 @@
 {{ config(materialized='view') }}
+
 with course_activities as (
-    select * from {{ ref('__mitxpro__user_courseactivities') }}
+    select * from {{ ref('stg__mitxonline__openedx__tracking_logs__user_activity') }}
+    where courserun_readable_id is not null
 )
 
 select
@@ -9,15 +11,9 @@ select
     , openedx_user_id
     , useractivity_event_source
     , useractivity_event_type
-    , useractivity_event_name
-    , useractivity_event_object
     , useractivity_timestamp
     , json_query(useractivity_event_object, 'lax $.id' omit quotes) as useractivity_video_id
     , json_query(useractivity_event_object, 'lax $.duration' omit quotes) as useractivity_video_duration
     , json_query(useractivity_event_object, 'lax $.currentTime' omit quotes) as useractivity_video_currenttime
 from course_activities
-where useractivity_event_type in (
-    'load_video', 'play_video', 'pause_video', 'stop_video', 'seek_video'
-    , 'speed_change_video', 'show_transcript', 'hide_transcript'
-    , 'edx.video.closed_captions.hidden', 'edx.video.closed_captions.shown'
-)
+where useractivity_event_type in ('play_video', 'pause_video', 'stop_video')

src/ol_dbt/models/intermediate/mitxpro/_int_mitxpro__models.yml

@@ -1279,6 +1279,7 @@ models:
   - name: coursetopic_parent_coursetopic_id
     description: int, id for the parent coursetopic for subtopics. For example 'Technology'
       is the parent topic for 'Technology:Data Science'
+
 - name: int__mitxpro__user_courseactivities
   description: xPro open edX user activities aggregated statistics per course from
     tracking logs
@@ -1313,9 +1314,113 @@ models:
     description: int, number of unique videos played within a course
   - name: courseactivity_last_play_video_timestamp
     description: timestamp, timestamp of user's last play_video event within a course
+  - name: courseactivity_last_problem_check_timestamp
+    description: timestamp, timestamp of user's last problem_check event within a
+      course
   tests:
   - dbt_expectations.expect_compound_columns_to_be_unique:
       column_list: ["user_username", "courserun_readable_id"]
+
+- name: int__mitxpro__user_courseactivity_video
+  description: It tracks xPro open edX users video activities within a course
+  columns:
+  - name: user_username
+    description: str, username of the open edX user who caused the event to be emitted.
+    tests:
+    - not_null
+  - name: openedx_user_id
+    description: int, reference user id in auth_user from open edX.
+    tests:
+    - not_null
+  - name: courserun_readable_id
+    description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}
+    tests:
+    - not_null
+  - name: useractivity_event_source
+    description: str, specifies the source of the interaction that triggered the event.
+      The values are either browser or mobile for this event
+    tests:
+    - not_null
+  - name: useractivity_event_type
+    description: str, type of video event triggered - play_video - When a user selects
+      the video player’s play control pause_video - When a user selects the video
+      player’s pause control stop_video - When the video player reaches the end of
+      the video file and play automatically stops
+    tests:
+    - not_null
+  - name: useractivity_video_id
+    description: str, hash code for the video being watched. This value is part of
+      the courseware_studentmodule.module_id
+    tests:
+    - not_null
+  - name: useractivity_video_duration
+    description: number, The length of the video file, in seconds.
+    tests:
+    - not_null
+  - name: useractivity_video_currenttime
+    description: number, The time in the video when this event was emitted
+  - name: useractivity_timestamp
+    description: timestamp, time when this event was emitted
+    tests:
+    - not_null
+  tests:
+  - dbt_expectations.expect_compound_columns_to_be_unique:
+      column_list: ["user_username", "courserun_readable_id", "useractivity_event_type",
+        "useractivity_video_id", "useractivity_video_currenttime", "useractivity_timestamp"]
+
+- name: int__mitxpro__user_courseactivity_problemcheck
+  description: It tracks xPro open edX users problem_check activities within a course
+  columns:
+  - name: user_username
+    description: str, username of the open edX user who caused the event to be emitted.
+    tests:
+    - not_null
+  - name: openedx_user_id
+    description: int, reference user id in auth_user from open edX.
+    tests:
+    - not_null
+  - name: courserun_readable_id
+    description: str, Open edX Course ID formatted as course-v1:{org}+{course code}+{run_tag}
+    tests:
+    - not_null
+  - name: useractivity_event_type
+    description: str, problem_check - when a problem is successfully checked.
+    tests:
+    - not_null
+  - name: useractivity_problem_id
+    description: str, Unique ID for this problem in a course. It's recorded as a URL
+      format - block-v1:{org)+{course ID}+type@problem+block@{hash code}
+    tests:
+    - not_null
+  - name: useractivity_problem_name
+    description: str, display name of this problem in a course
+    tests:
+    - not_null
+  - name: useractivity_problem_attempts
+    description: number, The number of times the user attempted to answer this problem
+    tests:
+    - not_null
+  - name: useractivity_problem_success
+    description: str, It's either 'correct' or 'incorrect'
+    tests:
+    - not_null
+  - name: useractivity_problem_current_grade
+    description: number, current grade value for this user
+    tests:
+    - not_null
+  - name: useractivity_problem_max_grade
+    description: number, Maximum possible grade value for this problem
+    tests:
+    - not_null
+  - name: useractivity_timestamp
+    description: timestamp, time when this event was emitted
+    tests:
+    - not_null
+  tests:
+  - dbt_expectations.expect_compound_columns_to_be_unique:
+      column_list: ["user_username", "courserun_readable_id", "useractivity_problem_id",
+        "useractivity_timestamp"]
+
 - name: int__mitxpro__courses_to_topics
   columns:
   - name: course_id