Merge branch 'main' of https://github.com/code-dot-org/analytics into fix/country_name_hoc_events

Author: jordan-springer

dbt/models/marts/teachers/dim_active_teachers.sql renamed to dbt/archive/dim_active_teachers.sql

@@ -1,6 +1,7 @@
 version: 2
 
 models: 
+  
   - name: dim_global_trained_teacher_script_progress
     description: This model contains various aggregations on the teacher/script/year level to measure student progress by particular scripts YoY for a given global trained teacher. 
     columns: 
@@ -116,92 +117,6 @@ models:
     config:
       tags: ['released']
 
-  - name: dim_active_teachers
-    description: |
-      This model serves as a prototype for a comprehensive view of daily teacher activities that roll into the 'active teacher' metric. 
-
-      A teacher is 'active' on a day if they sign-in or perform any one of the set of core teacher actions that we capture via Amplitude.
-      (for more details see - placeholder for doc - or the RED team).
-      
-      The model merges platform sign-in data with event data from Amplitude. For both sign-ins and various event interactions
-      captured via Amplitude, it captures both occurrence of these events (multiple events aggregated to a day), as well 
-      as geographical and user type information. Amplitude data can be joined to platform data via a Code.org user_id which 
-      Amplitude tracks once it learns that a user is a Code.org user. 
-      
-      In cases where we have data about a user from both Amplitude and Code.org, for example which country the user is from, we merge
-      it together giving preference to the Code.org version of that data.
-
-    columns:
-      - name: event_date
-        description: "The date on which either a sign-in or an Amplitude event was recorded (or both). This field merges dates from sign-ins and Amplitude events to align activities across sources."
-        data_type: "date"
-        data_tests:
-          - not_null
-
-      - name: merged_user_id
-        description: "A unified identifier for the user that merges user IDs from Code.org platform sign-ins and Amplitude events, ensuring consistent tracking across sources. This user_id will be the user's Code.org user_id if it is known or the amplitude_id if not."
-        data_type: "varchar"
-        data_tests:
-          - not_null
-
-      - name: user_type
-        description: "The type of the user, which can be 'teacher', 'anon', or other user types from Code.org or Amplitude data. If the user is of type = 'teacher' then the user is a *known code.org teacher user* User_type can occasionally be null for brand new user accounts, when amplitude data has a user_id recorded, but it hasn't been migrated into dim_users yet."
-        data_type: "varchar"
-
-      - name: country
-        description: "Country information for the user, merges Code.org geolocation and Amplitude geolocation data. Preference is given to the Code.org data in the case of a conflict between sources."
-        data_type: "varchar"
-
-      - name: us_intl
-        description: "Values are `us` or `intl`. Indicates whether the user's country (see: country) is in the US or non-us."
-        data_type: "varchar"
-
-      - name: cdo_user_id
-        description: "The Code.org user ID from the sign-in data."
-        data_type: "varchar"
-
-      - name: amplitude_id
-        description: "Amplitude's unique identifier for a user, used in tracking event data."
-        data_type: "varchar"
-
-      - name: num_cdo_records
-        description: "Count of sign-ins from Code.org data for the user."
-        data_type: "integer"
-
-      - name: num_amp_records
-        description: "The number of records from Amplitude for the user for this date."
-        data_type: "integer"
-
-      - name: num_records
-        description: "The total number of records from both Code.org sign-ins and Amplitude events for the user on this date."
-        data_type: "integer"
-
-      - name: event_sources
-        description: "Possible values: (`cdo`, 'amp,cdo`, `amp`). Comma-separated list indicating the sources of the events ('cdo' for Code.org and 'amp' for Amplitude)."
-        data_type: "varchar"
-
-      - name: events_list
-        description: | 
-          Comma-separated list of event names for activities teachers engaged in. This list aggregates events from both Code.org sign-ins and Amplitude data.
-          Code.org sign-in is listed as `cdo_sign_in`.
-          Amplitude events have had their names shortened according to the following
-          - '[Amplitude event name]' --> '[shortened version]'
-          - 'Teacher Viewing Student Work' --> 'View Work'
-          - 'Section Progress Viewed' -->  'View Progress'
-          - 'Teacher Login' -->  'Login Page'
-          - 'Unit Overview Page Visited By Teacher' --> 'View Unit Page'
-          - 'Lesson Overview Page Visited' -->  'View Lesson Plan'
-          - 'Section Progress Unit Changed' --> 'Change unit'
-        data_type: "text"
-
-    data_tests:
-      - dbt_utils.unique_combination_of_columns:
-          combination_of_columns:
-            - merged_user_id
-            - event_date
-            - user_type
-            - us_intl
-
   - name: dim_teachers
     description: |
       One row per teacher account ever created, with geographic information and current school association
@@ -256,5 +171,61 @@ models:
           combination_of_columns:
             - school_year
             - teacher_id
+    config:
+      tags: ['released']
+    
+  - name: dim_daily_teacher_activity
+    description: |
+      One row per teacher / activity date for every date a teacher had activity, where "activity" is defined by the following events tracked in Statsig (and their activity type classifications): 
+        curriculum catalog visited (light)    
+        level activity (light)     
+        lesson overview page visited (moderate)   
+        unit overview page visited by teacher (moderate)     
+        lesson resource link visited (moderate)         
+        teacher viewing student work (moderate)      
+        section setup completed (moderate)     
+        section curriculum assigned (moderate)            
+        section progress viewed (moderate) 
+        teacher viewing student work (heavy)  
+        level feedback submitted (heavy)   
+        rubric activity (heavy)  
+    columns: 
+      - name: teacher_id
+        description: unique ID for the teacher with the activity
+        data_tests: 
+          - not_null
+
+      - name: activity_date
+        description: the date on which the teacher had activity
+        data_tests: 
+          - not_null
+      
+      - name: school_year
+        description: the school year associated with the activity date, used for aggregation
+        data_tests: 
+          - not_null
+
+      - name: cal_year 
+        description: the calendar year associated with the activity date, used for aggregation
+        data_tests: 
+          - not_null
+      
+      - name: has_light_activity
+        description: 1 if the teacher did any event considered "light" on the activity date, 0 otherwise
+      
+      - name: has_moderate_activity
+        description: 1 if the teacher did any event considered "moderate" on the activity date, 0 otherwise
+      
+      - name: has_heavy_activity
+        description: 1 if the teacher did any event considered "heavy" on the activity date, 0 otherwise
+
+      - name: events_touched
+        description: comma separated list of events the teacher did on the activity date
+
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - teacher_id
+            - activity_date
     config:
       tags: ['released']

dbt/models/marts/metrics/fct_active_teachers_daily.sql renamed to dbt/archive/fct_active_teachers_daily.sql

@@ -0,0 +1,124 @@
+with 
+
+school_years as (
+    select * 
+    from {{ ref('int_school_years') }}
+),
+
+teachers as (
+    select distinct
+        teacher_id,
+        us_intl
+    from {{ ref('dim_teachers') }}
+),
+
+statsig_events as (
+    select * 
+    from {{ ref('stg_analysis_pii__statsig_events') }} statsig_events
+    join teachers on statsig_events.user_id = teachers.teacher_id
+    where event_name in (
+        'curriculum catalog visited',
+        'level activity',
+        'lesson overview page visited',
+        'unit overview page visited by teacher',
+        'lesson resource link visited',
+        'section setup completed',
+        'section curriculum assigned',
+        'teacher viewing student work',
+        'section progress viewed',
+        'level feedback submitted',
+        'rubric activity'
+    )
+),
+
+activity_levels as (
+    select 
+        user_id                                 as teacher_id,
+        us_intl,
+        trunc(event_at)                         as activity_date,
+        event_name,
+        case
+            when event_name in (
+                'curriculum catalog visited',
+                'level activity'
+            )
+            then 1
+            else 0
+        end                                     as has_light_activity,
+
+        case
+            when event_name in (
+                'lesson overview page visited',
+                'unit overview page visited by teacher',
+                'lesson resource link visited',
+                'section setup completed',
+                'section curriculum assigned',
+                'section progress viewed'
+            )
+            then 1
+            else 0
+        end                                     as has_moderate_activity,
+
+        case
+            when event_name in (
+                'teacher viewing student work',
+                'level feedback submitted',
+                'rubric activity'
+            )
+            then 1
+            else 0
+        end                                     as has_heavy_activity
+
+    from statsig_events
+    group by 1,2,3,4,5
+)
+
+select
+    teacher_id, 
+    us_intl,
+    activity_date,
+    school_years.school_year,
+    extract(year from activity_date)                 as cal_year,
+    case 
+        when sum(has_heavy_activity) >= 1
+        then 'heavy'
+    else 
+        case 
+            when sum(has_moderate_activity) >= 1
+            then 'moderate'
+        else 
+            case
+                when sum(has_light_activity) >= 1
+                then 'light'
+            else null 
+            end
+        end
+    end                                             as activity_type,
+
+    case 
+        when sum(has_light_activity) >= 1
+        then 1 
+        else 0 
+    end                                             as has_light_activity,
+    
+    case 
+        when sum(has_moderate_activity) >= 1
+        then 1 
+        else 0 
+    end                                             as has_moderate_activity,
+
+    case 
+        when sum(has_heavy_activity) >= 1
+        then 1 
+        else 0 
+    end                                             as has_heavy_activity,
+    
+    listagg(distinct event_name, ', ') 
+            within group (
+                order by teacher_id, activity_date
+            )                                       as events_touched
+
+from activity_levels 
+join school_years 
+    on activity_levels.activity_date between school_years.started_at and school_years.ended_at
+group by 1,2,3,4,5

dbt/models/marts/teachers/_teachers__models.yml

@@ -18,12 +18,13 @@ statsig_events as (
         cast(company_metadata.unitName as varchar)  as unit_name,
         cast(company_metadata.pageUrl as varchar)   as page_url 
     from {{ ref('base_analysis_pii__statsig_events') }} 
+    where event_at >= '2024-11-06' -- exclude the two october dates
 ),
 
 renamed as (
     select 
         user_id,
-        -- stable_id,	
+        stable_id,	
         custom_ids,	
         event_at,
         event_name,