Merge branch 'main' into feature/update_dim_user_levels_content_fields

dbt/data/_seed_files.yml

@@ -108,17 +108,33 @@ seeds:
       ```
 
   - name: seed_course_names
-  - name: seed_school_years
-  - name: seed_script_names
+
+  - name: seed_cs_state_grad_requirements
+    description: states that have passed CS grad requirements, current as of October 2024
+
   - name: seed_csf_plugged_stage_counts
     description: pulled manually from the output of what is currently in redshift
 
   - name: seed_csf_stages_for_completion
     description: pulled manually from the output of what is currently in redshift
 
-  - name: seed_training_school_years
+  - name: seed_districts_enrolled
+    description: districts enrolled in the district program, as of October 2024
+
+  - name: seed_districts_target
+    description: districts that are active targets for the district program, as of 2024-10-29
+
+  - name: seed_hoc_internal_tutorials
+    description: reference list of Code.org HOC tutorials (differentiated from 3rd party) provided by Bethany on 2024-10-29
 
+  - name: seed_school_years
+    description: calendar boundaries of school years
+
+  - name: seed_script_names
+
   - name: seed_state_abbreviations
     description: pulled manually from the output of what is currently in redshift
+
+  - name: seed_training_school_years
 
   - name: seed_workshop_state_zip_manual

dbt/data/seed_hoc_internal_tutorials.csv

@@ -0,0 +1,59 @@
+tutorial_codes,is_internal
+algorithmicbias-ai,1
+applab-intro,1
+aquatic,1
+artist,1
+basketball,1
+big_event,1
+chatbotsllm-ai,1
+codeintl,1
+codeofethics-ai,1
+codeorg,1
+computervision-ai,1
+dance-2019,1
+dance-ai-2023,1
+dance-extras-2019,1
+dance-extras,1
+dance,1
+disney_static_starwars,1
+disney_static_starwarsblocks,1
+flappy,1
+frozen,1
+generativeimages-ai,1
+getting_loopy,1
+graph_paper_programming,1
+gumball,1
+happy_maps,1
+hello-world-animals-2021,1
+hello-world-emoji-2021,1
+hello-world-food-2021,1
+hello-world-retro-2021,1
+hello-world-soccer-2022,1
+hello-world-space-2022,1
+hello-world,1
+hero,1
+hoc-encryption,1
+hourofcode,1
+iceage,1
+infinity,1
+introtoml-ai,1
+mc,1
+minecraft,1
+move_it,1
+music-jam-2024,1
+neuralnetworks-ai,1
+oceans,1
+outbreak,1
+persistence,1
+playlab,1
+poem_art,1
+poem-art-2021,1
+russia_mc,1
+socimpactofgenai-ai,1
+sports,1
+starwarsblocks,1
+static_mc,1
+static_starwars,1
+static_starwarsblocks,1
+sw,1
+text-compression,1

dbt/models/marts/hoc/_hoc__models.yml

@@ -6,7 +6,6 @@ models:
       this model contains 1 row for every URL hit of an HoC activity, regardless of whether the user was signed in or the user was on a 3rd party site.   
           
       note-- if you'd like to explore only signed-in hoc usage, you can use use dim_student_script_level_activity
-
     columns:
       - name: hoc_start_id
         data_tests: 
@@ -22,7 +21,7 @@ models:
       - name: company 
       - name: tutorial
       - name: is_third_party
-        description: 1 if the URL hit was from a third party site with pixel tracking, 0 otherwise
+        description: 1 if the tutorial matches to our manually maintained list of Code.org HOC tutorials. Last updated 11/25/24 for the 2024 HOC year. 
       - name: city
         description: the city associated with the user's location
       - name: country 

dbt/models/marts/hoc/dim_hoc_starts.sql

@@ -3,28 +3,44 @@ with
 hoc_activity as (
     select * 
     from {{ ref('stg_pegasus_pii__hoc_activity') }}
+    where hoc_start_id is not NULL
 ),
 
 school_years as (
     select * 
     from {{ ref('int_school_years') }}
-)
+),
+
+internal_tutorials as (
+    select *
+    from {{ ref('seed_hoc_internal_tutorials') }}
+),
 
-select 
-    hoc_activity.hoc_start_id
-    , hoc_activity.started_at
-    , sy.school_year_int                                                as cal_year 
-    , sy.school_year
-    -- , hoc_activity.referer
-    , hoc_activity.company
-    , hoc_activity.tutorial
-    , hoc_activity.is_third_party
-    , hoc_activity.city
-    , hoc_activity.country
-    , hoc_activity.state
-    , hoc_activity.state_code
-    , hoc_activity.country_code
-from hoc_activity 
-join school_years                                                       as sy 
-    on hoc_activity.started_at between sy.started_at and sy.ended_at
+final as (
+    select 
+        hoc_activity.hoc_start_id
+        , hoc_activity.started_at
+        , sy.school_year_int                                                as cal_year 
+        , sy.school_year
+        --, hoc_activity.referer
+        , hoc_activity.company
+        , hoc_activity.tutorial
+        , case
+            when it.is_internal is null then 1
+            else 0
+            end as is_third_party
+        , hoc_activity.city
+        , hoc_activity.country
+        , hoc_activity.state
+        , hoc_activity.state_code
+        , hoc_activity.country_code
+    from hoc_activity 
+    join school_years                                                       as sy 
+        on hoc_activity.started_at 
+            between sy.started_at 
+            and sy.ended_at
+    left join internal_tutorials                                            as it
+        on hoc_activity.tutorial = it.tutorial_codes )
 
+select *
+from final

dbt/models/staging/pegasus_pii/_pegasus_pii__models.yml

@@ -12,5 +12,17 @@ models:
   - name: stg_pegasus_pii__hoc_activity
     description: |
       Staging model for `hoc_activity` source data
+    columns:
+      - name: hoc_start_id
+      - name: referer
+      - name: company
+      - name: tutorial
+      - name: started_at 
+        description: coalesce(started_at, pixel_started_at, pixel_finished_at)
+      - name: country_code
+      - name: state_code
+      - name: city
+      - name: country
+      - name: state
 
 

dbt/models/staging/pegasus_pii/stg_pegasus_pii__hoc_activity.sql

@@ -13,15 +13,6 @@ hoc_starts as (
         company,
         tutorial,
         coalesce(started_at, pixel_started_at, pixel_finished_at)   as started_at,
-        case 
-            when 
-                coalesce(
-                    pixel_started_at
-                    , pixel_finished_at
-                ) is not null 
-            then 1 
-            else 0 
-        end                                                          as is_third_party,
         country_code,
         state_code,
         city,
@@ -31,10 +22,9 @@ hoc_starts as (
     {% if is_incremental() %}
 
     where coalesce(started_at, pixel_started_at, pixel_finished_at) > (select max(started_at) from {{ this }} )
-    
+
     {% endif %}
 )
 
 select *
-from hoc_starts
-
+from hoc_starts