From edaaf7ea44ddb269126a84dd45743fedea2fc6c0 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Thu, 29 Aug 2024 03:10:11 +0000 Subject: [PATCH 01/10] init --- .../students/dim_student_course_activity | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 dbt/models/marts/students/dim_student_course_activity diff --git a/dbt/models/marts/students/dim_student_course_activity b/dbt/models/marts/students/dim_student_course_activity new file mode 100644 index 00000000..a445cd8c --- /dev/null +++ b/dbt/models/marts/students/dim_student_course_activity @@ -0,0 +1,203 @@ +-- version: 2.0 (JS) + + +with +user_levels as ( +/* + 1. User Course Activity + + This is the mostly costly part of the model, so we will + build it as easily as possible and earlier in the process... + + This first combination allows us to pull together all the user level activity and course data we need to build the rest of the model. + + We can also roll through our aggregates and other calculations so as to not do them each time in some larger downstream query... +*/ + select + -- keys + + user_id as student_id, + -- If it's a student model, let's maintain prior conventions + + level_id, + script_id, + + -- dates + date_trunc('day', created_at) as activity_date, + extract('month' from created_at) as activity_month, + + -- aggs + max(attempts) as total_attempts, + max(best_result) as best_result, + sum(time_spent) as time_spent_minutes + from {{ ref('stg_dashboard__user_levels') }} + {{ dbt_utils.group_by(3) }} +), + +course_structure as ( + select + course_name, + course_id, + level_id, + level_name, + level_type, + unit as unit_name, + stage_name as lesson_name + from {{ ref('dim_course_structure') }} + + where participant_audience = 'student' + -- Note: filter out data we don't want or need as early as possible. If we keep it around, it will be continuously processed as it is referenced in other queries. +), + +school_years as ( + select * + from {{ ref('int_school_years') }} +), + +student_activity as ( + select + ul.* , + sy.school_year + + -- calc for qtr + case + when ul.activity_month in (7,8,9) then 'Q1' + when ul.activity_month in (10,11,12) then 'Q2' + when ul.activity_month in (1,2,3) then 'Q3' + when ul.activity_month in (4,5,6) then 'Q4' + end as activity_quarter, + + cs.course_name, + cs.level_name, + cs.level_type, + cs.unit_name, + cs.lesson_name + + from user_levels as ul + left join course_structure as cs + on ul.level_id = cs.level_id + and ul.script_id = cs.script_id + join school_years as sy + on comb.activity_date + between sy.start_date + and sy.end_date +), + +/* + 2. Sections and Students + + We now have all that taken care of, so our join will be simpler later on (and less data in memory) + + Next, we need to map each student to their section(s). This is another big one but only bc our student_activity isn't unique to student_id... so we can pre-process out that work as well. +*/ + +section_mapping as ( + select * + from {{ ref('int_section_mapping') }} + + where student_id in (select user_id from combined) + -- Note: again, filter out anything we don't need. We need to keep the ship as light as possible. +), + +section_size as ( + select + section_id, + count(distinct student_id) as section_size + from section_mapping +), + +sections as ( + select + scm.*, + -- Note: I don't actually need student_activity data here, so I won't bother to load it + + scz.section_size + from section_mapping as scm + join section_size as scz + on scm.section_id = scz.section_id + join student_activity as sta + on scm.student_id = sta.student_id + and scm.school_year = sta.school_year +), + +/* + So now that we have our student_activity and section mapping built, we can smash them together to get our full sections and status data + + 3. Teachers and Schools + a. Use student_id to connect to sections + b. Use teacher, school_id for statuses + c. select * that shit + d. almost forgot user_geo info +*/ + +schools as ( + select * + from {{ ref('dim_schools') }} +), + +users as ( + select user_id, is_international, country + from {{ ref('dim_users') }} +), + +school_status as ( + select school_id, school_year, school_status + from {{ ref('dim_school_status') }} +), + +teacher_status as ( + select teacher_id, school_year, status as teacher_status + from {{ ref('dim_teacher_status') }} +), + +combined as ( + select + sec.school_year, + + -- teachers + sec.teacher_id, + tes.teacher_status, + + -- schools + sec.school_id, + sch.school_status, + sch.school_name as school_name, + sch.school_district_id as school_district_id, + sch.school_district_name as school_district_name, + sch.state as school_state, + sch.school_type as school_type, + sch.is_stage_el as school_is_stage_el, + sch.is_stage_mi as school_is_stage_mi, + sch.is_stage_hi as school_is_stage_hi, + sch.is_high_needs as school_is_high_needs, + sch.is_rural as school_is_rural, + + usr.student_id, + usr.is_international, + usr.country + + from sections as sec + + join teacher_status as tes + on tes.teacher_id = sec.teacher_id + and tes.school_year = sec.school_year + + join school_status as sst + on sec.school_id = sst.school_id + and sec.school_year = sst.school_year + + join schools as sch + on sch.school_id = sec.school_id + join users as usr + on usr.user_id = sec.student_id +), + +final as ( + select * + from combined + cross join student_activity + using student_id, school_year +) + +select * from final +limit 1 \ No newline at end of file From c6be69cd32424b6eca2386f61d280bd78a0d325b Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Fri, 30 Aug 2024 16:37:42 +0000 Subject: [PATCH 02/10] fingers crossed --- .../dim_student_script_level_activity.sql | 0 .../marts/students/_students__models.yml | 19 +++---- ...tivity => dim_student_course_activity.sql} | 52 +++++++++---------- 3 files changed, 32 insertions(+), 39 deletions(-) rename dbt/{models/marts => dev}/students/dim_student_script_level_activity.sql (100%) rename dbt/models/marts/students/{dim_student_course_activity => dim_student_course_activity.sql} (86%) diff --git a/dbt/models/marts/students/dim_student_script_level_activity.sql b/dbt/dev/students/dim_student_script_level_activity.sql similarity index 100% rename from dbt/models/marts/students/dim_student_script_level_activity.sql rename to dbt/dev/students/dim_student_script_level_activity.sql diff --git a/dbt/models/marts/students/_students__models.yml b/dbt/models/marts/students/_students__models.yml index 4491f267..b4bc9bb8 100644 --- a/dbt/models/marts/students/_students__models.yml +++ b/dbt/models/marts/students/_students__models.yml @@ -182,7 +182,8 @@ models: config: tags: ['released'] - - name: dim_student_script_level_activity +# - name: dim_student_script_level_activity + - name: dim_student_course_activity description: | This model contains 1 row for every level/script touched by a student. This model also includes important segmentations (e.g. school status at the time of activity, whether or not the student was in a section at that time, etc) @@ -248,16 +249,8 @@ models: - name: time_spent_minutes description: the number of minutes the student spent on that level on the particular date config: - tags: ['released'] + tags: ['released'] #} -# - name: dim_student_status -# description: This model categorizes students based on their activity status across different school years. It provides insights into user engagement by assigning a status that reflects their activity in the current, previous, and any earlier school years. -# columns: -# - name: student_id -# description: the unique ID associated with the activity -# - name: school_year -# description: school year in which an activity status is assigned -# - name: status -# description: the activity status of a student in a given school year- active if they have attempted at least 1 level of a course -# - name: courses_started -# description: comma separated list of courses associated with the levels attemped by the student in a given school year + - name: dim_student_course_activity + description: One row per student, script/level (course), and activity date + diff --git a/dbt/models/marts/students/dim_student_course_activity b/dbt/models/marts/students/dim_student_course_activity.sql similarity index 86% rename from dbt/models/marts/students/dim_student_course_activity rename to dbt/models/marts/students/dim_student_course_activity.sql index a445cd8c..ee2332a0 100644 --- a/dbt/models/marts/students/dim_student_course_activity +++ b/dbt/models/marts/students/dim_student_course_activity.sql @@ -14,8 +14,6 @@ user_levels as ( We can also roll through our aggregates and other calculations so as to not do them each time in some larger downstream query... */ select - -- keys - user_id as student_id, -- If it's a student model, let's maintain prior conventions @@ -31,13 +29,14 @@ user_levels as ( max(best_result) as best_result, sum(time_spent) as time_spent_minutes from {{ ref('stg_dashboard__user_levels') }} - {{ dbt_utils.group_by(3) }} + {{ dbt_utils.group_by(5) }} ), course_structure as ( select course_name, course_id, + script_id, level_id, level_name, level_type, @@ -57,7 +56,7 @@ school_years as ( student_activity as ( select ul.* , - sy.school_year + sy.school_year, -- calc for qtr case @@ -78,9 +77,9 @@ student_activity as ( on ul.level_id = cs.level_id and ul.script_id = cs.script_id join school_years as sy - on comb.activity_date - between sy.start_date - and sy.end_date + on ul.activity_date + between sy.started_at + and sy.ended_at ), /* @@ -95,7 +94,7 @@ section_mapping as ( select * from {{ ref('int_section_mapping') }} - where student_id in (select user_id from combined) + where student_id in (select student_id from student_activity) -- Note: again, filter out anything we don't need. We need to keep the ship as light as possible. ), @@ -104,11 +103,12 @@ section_size as ( section_id, count(distinct student_id) as section_size from section_mapping + {{ dbt_utils.group_by(1) }} ), sections as ( select - scm.*, + scm.* , -- Note: I don't actually need student_activity data here, so I won't bother to load it scz.section_size @@ -141,7 +141,7 @@ users as ( ), school_status as ( - select school_id, school_year, school_status + select school_id, school_year, status as school_status from {{ ref('dim_school_status') }} ), @@ -157,24 +157,25 @@ combined as ( -- teachers sec.teacher_id, tes.teacher_status, + + -- students + sec.student_id, + usr.is_international, + usr.country, -- schools sec.school_id, - sch.school_status, - sch.school_name as school_name, - sch.school_district_id as school_district_id, - sch.school_district_name as school_district_name, + sst.school_status, + sch.school_name, + sch.school_district_id, + sch.school_district_name, sch.state as school_state, - sch.school_type as school_type, + sch.school_type, sch.is_stage_el as school_is_stage_el, sch.is_stage_mi as school_is_stage_mi, sch.is_stage_hi as school_is_stage_hi, sch.is_high_needs as school_is_high_needs, - sch.is_rural as school_is_rural, - - usr.student_id, - usr.is_international, - usr.country + sch.is_rural as school_is_rural from sections as sec @@ -194,10 +195,9 @@ combined as ( final as ( select * - from combined - cross join student_activity - using student_id, school_year -) + from combined as comb + left join student_activity as sta + on comb.student_id = sta.student_id ) -select * from final -limit 1 \ No newline at end of file +select * +from final \ No newline at end of file From d12835f686d0077513f72890378271e4de3bd4b1 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Fri, 30 Aug 2024 18:49:43 +0000 Subject: [PATCH 03/10] moar --- dbt/dev/hoc/_hoc__models.yml | 19 +++- .../students/dim_student_projects.sql | 0 dbt/models/marts/hoc/_hoc__models.yml | 32 +++--- dbt/models/marts/hoc/fct_hoc_activity.sql | 96 ----------------- .../marts/students/_students__models.yml | 100 +++++++++--------- 5 files changed, 84 insertions(+), 163 deletions(-) rename dbt/{models/marts => dev}/students/dim_student_projects.sql (100%) delete mode 100644 dbt/models/marts/hoc/fct_hoc_activity.sql diff --git a/dbt/dev/hoc/_hoc__models.yml b/dbt/dev/hoc/_hoc__models.yml index ca780e5b..f9409070 100644 --- a/dbt/dev/hoc/_hoc__models.yml +++ b/dbt/dev/hoc/_hoc__models.yml @@ -1,4 +1,21 @@ version: 2 + models: - name: fct_hoc_activity - description: commonly tracked hoc metrics by hoc year and country \ No newline at end of file + description: | + commonly tracked hoc metrics by month, school year, and country + + columns: + - name: month + - name: school_year + - name: country + description: the country associated with the user's location + - name: total_new_teacher_accounts + description: the number of new teacher accounts created in the given month + - name: total_new_student_accounts + description: the number of new student accounts created in the given month + - name: total_event_registrations + description: the number of HoC events registered in the given month + - name: total_hoc_hits + description: the number of times an HoC activity was accessed + \ No newline at end of file diff --git a/dbt/models/marts/students/dim_student_projects.sql b/dbt/dev/students/dim_student_projects.sql similarity index 100% rename from dbt/models/marts/students/dim_student_projects.sql rename to dbt/dev/students/dim_student_projects.sql diff --git a/dbt/models/marts/hoc/_hoc__models.yml b/dbt/models/marts/hoc/_hoc__models.yml index 153818d5..325a2c32 100644 --- a/dbt/models/marts/hoc/_hoc__models.yml +++ b/dbt/models/marts/hoc/_hoc__models.yml @@ -1,23 +1,23 @@ version: 2 models: - - name: fct_hoc_activity - description: | - commonly tracked hoc metrics by month, school year, and country +# - name: fct_hoc_activity +# description: | +# commonly tracked hoc metrics by month, school year, and country - columns: - - name: month - - name: school_year - - name: country - description: the country associated with the user's location - - name: total_new_teacher_accounts - description: the number of new teacher accounts created in the given month - - name: total_new_student_accounts - description: the number of new student accounts created in the given month - - name: total_event_registrations - description: the number of HoC events registered in the given month - - name: total_hoc_hits - description: the number of times an HoC activity was accessed +# columns: +# - name: month +# - name: school_year +# - name: country +# description: the country associated with the user's location +# - name: total_new_teacher_accounts +# description: the number of new teacher accounts created in the given month +# - name: total_new_student_accounts +# description: the number of new student accounts created in the given month +# - name: total_event_registrations +# description: the number of HoC events registered in the given month +# - name: total_hoc_hits +# description: the number of times an HoC activity was accessed - name: dim_hoc_starts description: | diff --git a/dbt/models/marts/hoc/fct_hoc_activity.sql b/dbt/models/marts/hoc/fct_hoc_activity.sql deleted file mode 100644 index 74435828..00000000 --- a/dbt/models/marts/hoc/fct_hoc_activity.sql +++ /dev/null @@ -1,96 +0,0 @@ -with - -hoc_starts as ( - select * - from {{ ref('dim_hoc_starts') }} -), - -school_years as ( - select * - from {{ ref('int_school_years') }} -), - -forms_hoc as ( - select * - from {{ ref('dim_hoc_event_registrations') }} -), - --- form_geos as ( --- select * --- from {{ ref("stg_pegasus_pii__form_geos") }} --- ), - -users as ( - select * - from {{ ref("stg_dashboard__users") }} -), - -user_geos as ( - select * - from {{ ref("stg_dashboard__user_geos") }} -), - -hoc_hits as ( - select - date_trunc('month', hoc.started_at) as hoc_month, - hoc.school_year, - hoc.country, - count(distinct hoc.hoc_start_id) as total_hoc_hits - - from hoc_starts as hoc - group by - hoc_month, - hoc.school_year, - hoc.country -), - -hoc_event_reg as ( - select - date_trunc('month', forms_hoc.registered_at) as registration_month, - forms_hoc.school_year, - forms_hoc.country, - count(distinct forms_hoc.form_id) as total_event_registrations - from forms_hoc - -- left join form_geos - -- on forms_hoc.form_id = form_geos.form_id - group by - registration_month, - forms_hoc.school_year, - forms_hoc.country -), - -new_accounts as ( - select - date_trunc('month', users.created_at) as created_month, - sy.school_year, - user_geos.country, - sum(case when user_type = 'teacher' then 1 end) as total_new_teacher_accounts, - sum(case when user_type = 'student' then 1 end) as total_new_student_accounts - from users - left join user_geos - on users.user_id = user_geos.user_id - left join school_years as sy - on users.created_at between sy.started_at and sy.ended_at - where current_sign_in_at is not null - group by - created_month, - sy.school_year, - user_geos.country -) - -select na.created_month as month_of, - na.school_year, - na.country, - na.total_new_teacher_accounts, - na.total_new_student_accounts, - reg.total_event_registrations, - hh.total_hoc_hits -from new_accounts na -left join hoc_event_reg reg - on na.created_month = reg.registration_month - and na.school_year = reg.school_year - and na.country = reg.country -left join hoc_hits hh - on hh.hoc_month = reg.registration_month - and hh.school_year = reg.school_year - and hh.country = reg.country \ No newline at end of file diff --git a/dbt/models/marts/students/_students__models.yml b/dbt/models/marts/students/_students__models.yml index b4bc9bb8..cb7814d3 100644 --- a/dbt/models/marts/students/_students__models.yml +++ b/dbt/models/marts/students/_students__models.yml @@ -1,73 +1,73 @@ version: 2 models: - - name: dim_student_projects - description: | - This model tracks all projects started on code.org. It includes additional information about the project itself, as well as other variables to make for easy segmentation for commonly asked questions. - columns: - - name: project_id - description: unique ID for the project - data_tests: - - not_null - - unique - - name: user_id - description: unique ID for the user, if they were signed in when completing the project - - - name: is_signed_in - description: 1 if the user was signed in while completing the project, 0 otherwise - - - name: user_type - description: student or teacher + # - name: dim_student_projects + # description: | + # This model tracks all projects started on code.org. It includes additional information about the project itself, as well as other variables to make for easy segmentation for commonly asked questions. + # columns: + # - name: project_id + # description: unique ID for the project + # data_tests: + # - not_null + # - unique + # - name: user_id + # description: unique ID for the user, if they were signed in when completing the project + + # - name: is_signed_in + # description: 1 if the user was signed in while completing the project, 0 otherwise + + # - name: user_type + # description: student or teacher - - name: is_published - description: 1 if the project was published, 0 otherwise + # - name: is_published + # description: 1 if the project was published, 0 otherwise - - name: project_created_at - description: the timestamp for when the project was first created + # - name: project_created_at + # description: the timestamp for when the project was first created - - name: project_updated_at - description: the timestamp for when the project was last updated + # - name: project_updated_at + # description: the timestamp for when the project was last updated - - name: project_published_at - description: the timestamp for when the project was published, null if not published + # - name: project_published_at + # description: the timestamp for when the project was published, null if not published - - name: school_year - description: the school year when the project was first created + # - name: school_year + # description: the school year when the project was first created - - name: cal_year - description: the calendar year when the project was first created + # - name: cal_year + # description: the calendar year when the project was first created - - name: country - description: the country associated with the signed-in user's location + # - name: country + # description: the country associated with the signed-in user's location - - name: intl_partner_id - description: the intl partner ID associated with user's country + # - name: intl_partner_id + # description: the intl partner ID associated with user's country - - name: intl_partner_name - description: the intl partner name associated with user's country + # - name: intl_partner_name + # description: the intl partner name associated with user's country - - name: region - description: the signed-in user's region + # - name: region + # description: the signed-in user's region - - name: is_standalone - description: 1 if the project was done as a standalone project, 0 if done as part of a curriculum + # - name: is_standalone + # description: 1 if the project was done as a standalone project, 0 if done as part of a curriculum - - name: abuse_score - description: the score given for reporting abuse in a project + # - name: abuse_score + # description: the score given for reporting abuse in a project - - name: project_type + # - name: project_type - - name: is_deleted - description: 1 if the project has been deleted, 0 if still active + # - name: is_deleted + # description: 1 if the project has been deleted, 0 if still active - - name: remix_parent_id - description: null if not remixed + # - name: remix_parent_id + # description: null if not remixed - - name: is_valid - description: 0 if the project is a "ghost project" with a null channel id, 1 otherwise + # - name: is_valid + # description: 0 if the project is a "ghost project" with a null channel id, 1 otherwise - config: - tags: ['released'] + # config: + # tags: ['released'] - name: dim_active_students description: | From d07f114ceb1f127d3b06d987f49b022049c34a29 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Fri, 30 Aug 2024 21:46:29 +0000 Subject: [PATCH 04/10] moar --- .../students/dim_student_course_activity.sql | 109 +++++++++++++----- 1 file changed, 80 insertions(+), 29 deletions(-) diff --git a/dbt/models/marts/students/dim_student_course_activity.sql b/dbt/models/marts/students/dim_student_course_activity.sql index ee2332a0..19580feb 100644 --- a/dbt/models/marts/students/dim_student_course_activity.sql +++ b/dbt/models/marts/students/dim_student_course_activity.sql @@ -1,8 +1,6 @@ -- version: 2.0 (JS) - with -user_levels as ( /* 1. User Course Activity @@ -13,9 +11,16 @@ user_levels as ( We can also roll through our aggregates and other calculations so as to not do them each time in some larger downstream query... */ +user_levels as ( select - user_id as student_id, - -- If it's a student model, let's maintain prior conventions + user_id, + + -- Note: + -- In order to not cross-polenate too soon, this part + -- of the model will maintain user_id only + -- Optional: + -- Create surrogate key for incremental load + -- md5 ( concat (user_id,activity_date,level_id,script_id ) as surrogate_key level_id, script_id, @@ -55,27 +60,42 @@ school_years as ( student_activity as ( select - ul.* , + + ul.user_id, + ul.level_id, + ul.script_id, + + -- dates sy.school_year, + ul.activity_date, + ul.activity_month, - -- calc for qtr + -- Note: post-process this work now that it is looking + -- for only a few values case - when ul.activity_month in (7,8,9) then 'Q1' - when ul.activity_month in (10,11,12) then 'Q2' - when ul.activity_month in (1,2,3) then 'Q3' - when ul.activity_month in (4,5,6) then 'Q4' + when ul.activity_month in ( 7, 8, 9 ) then 'Q1' + when ul.activity_month in ( 10, 11, 12 ) then 'Q2' + when ul.activity_month in ( 1, 2, 3 ) then 'Q3' + when ul.activity_month in ( 4, 5, 6 ) then 'Q4' end as activity_quarter, cs.course_name, cs.level_name, cs.level_type, cs.unit_name, - cs.lesson_name + cs.lesson_name, + + -- aggs + ul.total_attempts, + ul.best_result, + ul.time_spent_minutes from user_levels as ul + left join course_structure as cs on ul.level_id = cs.level_id and ul.script_id = cs.script_id + join school_years as sy on ul.activity_date between sy.started_at @@ -93,9 +113,11 @@ student_activity as ( section_mapping as ( select * from {{ ref('int_section_mapping') }} - - where student_id in (select student_id from student_activity) - -- Note: again, filter out anything we don't need. We need to keep the ship as light as possible. + + where student_id in ( + select user_id + from student_activity ) + -- Note: again, filter out anything we don't need. We need to keep the ship as light as possible. ), section_size as ( @@ -108,16 +130,12 @@ section_size as ( sections as ( select - scm.* , - -- Note: I don't actually need student_activity data here, so I won't bother to load it - + scm.*, scz.section_size + from section_mapping as scm join section_size as scz - on scm.section_id = scz.section_id - join student_activity as sta - on scm.student_id = sta.student_id - and scm.school_year = sta.school_year + on scm.section_id = scz.section_id ), /* @@ -152,17 +170,23 @@ teacher_status as ( combined as ( select - sec.school_year, - - -- teachers - sec.teacher_id, - tes.teacher_status, - -- students sec.student_id, + sec.school_year, + + case when sec.student_removed_at is not null + then 1 else 0 end as is_removed, + usr.is_international, usr.country, + -- teachers + sec.teacher_id, + tes.teacher_status, + + -- sections + sec.section_id, + -- schools sec.school_id, sst.school_status, @@ -177,6 +201,11 @@ combined as ( sch.is_high_needs as school_is_high_needs, sch.is_rural as school_is_rural + -- dates if need + + -- sec.student_added_at, + -- sec.student_removed_at, + from sections as sec join teacher_status as tes @@ -194,10 +223,32 @@ combined as ( ), final as ( - select * + select + comb.*, + + -- moar dates + sta.activity_date, + sta.activity_month, + sta.activity_quarter, + + -- coursework + sta.level_id, + sta.script_id, + sta.course_name, + sta.level_name, + sta.level_type, + sta.unit_name, + sta.lesson_name, + + -- totals + sta.total_attempts, + sta.best_result, + sta.time_spent_minutes + from combined as comb left join student_activity as sta - on comb.student_id = sta.student_id ) + on comb.student_id = sta.user_id + and comb.school_year = sta.school_year ) select * from final \ No newline at end of file From 5850aee79fe05afcf879c464e93c0f7ec8409df5 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Fri, 30 Aug 2024 21:51:18 +0000 Subject: [PATCH 05/10] moar --- .../marts/students/_students__models.yml | 38 +++++++++--- .../students/dim_student_course_activity.sql | 4 +- .../marts/teachers/_teachers__models.yml | 60 +++++++++---------- dbt/models/marts/users/_users__models.yml | 16 ++--- 4 files changed, 69 insertions(+), 49 deletions(-) diff --git a/dbt/models/marts/students/_students__models.yml b/dbt/models/marts/students/_students__models.yml index cb7814d3..ca2e3b6e 100644 --- a/dbt/models/marts/students/_students__models.yml +++ b/dbt/models/marts/students/_students__models.yml @@ -182,7 +182,6 @@ models: config: tags: ['released'] -# - name: dim_student_script_level_activity - name: dim_student_course_activity description: | This model contains 1 row for every level/script touched by a student. @@ -192,65 +191,86 @@ models: description: the unique ID associated with the activity data_tests: - not_null + - name: level_id description: the level the student was interacting with data_tests: - not_null + - name: script_id description: the script ID associated with the level the student was interacting with + - name: activity_date description: the date of the level-script activity, truncated to the day. - - name: school_year_quarter - description: the quarter of the school year in which the activity occurred (Q1 = Jul - Sep, Q2 = Oct - Dec, Q3 = Jan - Mar, Q4 = Apr - Jun) + - name: activity_school_year description: the school year in which the activity occurred + - name: level_name + - name: level_type + - name: unit_name description: formerly known as script_name + - name: course_name + - name: lesson_name + - name: section_id description: only populated if the student is in a section that school year + - name: section_teacher_id - descirption: the teacher id associated with the section (if the student is in a section) + description: the teacher id associated with the section (if the student is in a section) + - name: section_size description: the number of students in the student's section in that school year + - name: teacher_status description: the activity status of the teacher associated with the section in that school year of activity + - name: school_status description: the activity status of the school associated with the section/teacher in that school year of activity; only populated if student in a section with a teacher with an NCES school association + - name: school_name description: the school associated with the teacher of the student's section in the given school year; only populated if student in a section with a teacher with an NCES school association - name: school_district_id description: only populated if student in a section with a teacher with an NCES school association + - name: school_district_name description: only populated if student in a section with a teacher with an NCES school association + - name: school_state description: only populated if student in a section with a teacher with an NCES school association + - name: school_type description: only populated if student in a section with a teacher with an NCES school association + - name: school_is_stage_el description: 1 if the school associated with the section's teacher has grades K-5; only populated if student in a section with a teacher with an NCES school association + - name: school_is_stage_mi description: 1 if the school associated with the section's teacher has grades 6-8; only populated if student in a section with a teacher with an NCES school association + - name: school_is_stage_hi description: 1 if the school associated with the section's teacher has grades 9-12; only populated if student in a section with a teacher with an NCES school association + - name: school_is_high_needs description: only populated if student in a section with a teacher with an NCES school association - name: school_is_rural description: only populated if student in a section with a teacher with an NCES school association + - name: activity_country description: the country associated with the IP address of the activity (not the school association) + - name: total_attempts description: the total number of attempts the student had on the particular level on that date + - name: best_result description: the best result the student got in the particular level on that date + - name: time_spent_minutes description: the number of minutes the student spent on that level on the particular date - config: - tags: ['released'] #} - - - name: dim_student_course_activity - description: One row per student, script/level (course), and activity date + config: + tags: ['released'] #} \ No newline at end of file diff --git a/dbt/models/marts/students/dim_student_course_activity.sql b/dbt/models/marts/students/dim_student_course_activity.sql index 19580feb..8a2cb071 100644 --- a/dbt/models/marts/students/dim_student_course_activity.sql +++ b/dbt/models/marts/students/dim_student_course_activity.sql @@ -111,7 +111,7 @@ student_activity as ( */ section_mapping as ( - select * + select * from {{ ref('int_section_mapping') }} where student_id in ( @@ -181,7 +181,7 @@ combined as ( usr.country, -- teachers - sec.teacher_id, + sec.teacher_id as section_teacher_id, tes.teacher_status, -- sections diff --git a/dbt/models/marts/teachers/_teachers__models.yml b/dbt/models/marts/teachers/_teachers__models.yml index 23c4bcef..69e3094c 100644 --- a/dbt/models/marts/teachers/_teachers__models.yml +++ b/dbt/models/marts/teachers/_teachers__models.yml @@ -1,36 +1,36 @@ version: 2 models: - - name: dim_intl_teacher_roster - description: This model consists of all the form submissions by international teachers for professional development. - columns: - - name: teacher_id - - name: form_submitted_at - - name: cal_year - description: the calendar year that the form was submitted in - - name: first_name - - name: last_name - - name: pref_name - - name: email - - name: email_alt - - name: school_department - - name: school_municipality - - name: school_name - - name: school_city - - name: school_country - - name: workshop_date - - name: workshop_organizer - - name: workshop_course_name - - name: email_opt_in - description: 1 if the teacher opted into emails, 0 otherwise - - name: account_created_at - description: the date time on which the teacher's account was created on our platform - - name: current_sign_in_at - description: the most recent date time the teacher signed into our platform - - name: sign_in_count - description: the number of times the teacher has signed into our platform all-time - config: - tags: ['released'] + # - name: dim_intl_teacher_roster + # description: This model consists of all the form submissions by international teachers for professional development. + # columns: + # - name: teacher_id + # - name: form_submitted_at + # - name: cal_year + # description: the calendar year that the form was submitted in + # - name: first_name + # - name: last_name + # - name: pref_name + # - name: email + # - name: email_alt + # - name: school_department + # - name: school_municipality + # - name: school_name + # - name: school_city + # - name: school_country + # - name: workshop_date + # - name: workshop_organizer + # - name: workshop_course_name + # - name: email_opt_in + # description: 1 if the teacher opted into emails, 0 otherwise + # - name: account_created_at + # description: the date time on which the teacher's account was created on our platform + # - name: current_sign_in_at + # description: the most recent date time the teacher signed into our platform + # - name: sign_in_count + # description: the number of times the teacher has signed into our platform all-time + # config: + # tags: ['released'] - name: dim_self_paced_pd_activity description: | diff --git a/dbt/models/marts/users/_users__models.yml b/dbt/models/marts/users/_users__models.yml index 252557d2..c4984dbd 100644 --- a/dbt/models/marts/users/_users__models.yml +++ b/dbt/models/marts/users/_users__models.yml @@ -66,14 +66,14 @@ models: # data_tests: # - not_null - - name: dim_user_levels - description: | - Adds more columns to the base user_levels table. - columns: - - name: user_id - description: the unique user ID associated with the activity - data_tests: - - not_null + # - name: dim_user_levels + # description: | + # Adds more columns to the base user_levels table. + # columns: + # - name: user_id + # description: the unique user ID associated with the activity + # data_tests: + # - not_null - name: dim_user_course_activity description: | From 89dc30822523eacc97f5dd83091f6435d5f42351 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Sat, 31 Aug 2024 02:55:43 +0000 Subject: [PATCH 06/10] Testing --- dbt/models/marts/students/dim_student_course_activity.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/models/marts/students/dim_student_course_activity.sql b/dbt/models/marts/students/dim_student_course_activity.sql index 8a2cb071..2bc975ea 100644 --- a/dbt/models/marts/students/dim_student_course_activity.sql +++ b/dbt/models/marts/students/dim_student_course_activity.sql @@ -1,4 +1,5 @@ -- version: 2.0 (JS) +-- 2024-08-30 with /* From 3e8b305a19c378e3d34f5740b341b66f8e4e1054 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Wed, 4 Sep 2024 16:59:14 +0000 Subject: [PATCH 07/10] fixed tests --- dbt/models/marts/students/_students__models.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/models/marts/students/_students__models.yml b/dbt/models/marts/students/_students__models.yml index ca2e3b6e..97ff8afb 100644 --- a/dbt/models/marts/students/_students__models.yml +++ b/dbt/models/marts/students/_students__models.yml @@ -194,8 +194,8 @@ models: - name: level_id description: the level the student was interacting with - data_tests: - - not_null + # data_tests: + # - not_null - name: script_id description: the script ID associated with the level the student was interacting with From 81a2004e9d64f05eebb99828e180300669e35d85 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Mon, 9 Sep 2024 18:28:03 +0000 Subject: [PATCH 08/10] renaming models --- dbt/models/marts/students/_students__models.yml | 2 +- ...ourse_activity.sql => dim_student_script_level_activity.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename dbt/models/marts/students/{dim_student_course_activity.sql => dim_student_script_level_activity.sql} (100%) diff --git a/dbt/models/marts/students/_students__models.yml b/dbt/models/marts/students/_students__models.yml index 149b169b..83ee7985 100644 --- a/dbt/models/marts/students/_students__models.yml +++ b/dbt/models/marts/students/_students__models.yml @@ -182,7 +182,7 @@ models: config: tags: ['released'] - - name: dim_student_course_activity + - name: dim_student_script_level_activity description: | This model contains 1 row for every level/script touched by a student. This model also includes important segmentations (e.g. school status at the time of activity, whether or not the student was in a section at that time, etc) diff --git a/dbt/models/marts/students/dim_student_course_activity.sql b/dbt/models/marts/students/dim_student_script_level_activity.sql similarity index 100% rename from dbt/models/marts/students/dim_student_course_activity.sql rename to dbt/models/marts/students/dim_student_script_level_activity.sql From d25024bd7c8ce31f4dbafc19daea6ae284a661bd Mon Sep 17 00:00:00 2001 From: "allison@code.org" Date: Mon, 9 Sep 2024 19:31:09 +0000 Subject: [PATCH 09/10] adding script name --- dbt/models/marts/students/_students__models.yml | 2 ++ .../marts/students/dim_student_script_level_activity.sql | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dbt/models/marts/students/_students__models.yml b/dbt/models/marts/students/_students__models.yml index 83ee7985..4ea4d488 100644 --- a/dbt/models/marts/students/_students__models.yml +++ b/dbt/models/marts/students/_students__models.yml @@ -210,6 +210,8 @@ models: - name: level_type + - name: script_name + - name: unit_name description: formerly known as script_name diff --git a/dbt/models/marts/students/dim_student_script_level_activity.sql b/dbt/models/marts/students/dim_student_script_level_activity.sql index 2bc975ea..1aa4196a 100644 --- a/dbt/models/marts/students/dim_student_script_level_activity.sql +++ b/dbt/models/marts/students/dim_student_script_level_activity.sql @@ -83,6 +83,7 @@ student_activity as ( cs.course_name, cs.level_name, cs.level_type, + cs.script_name, cs.unit_name, cs.lesson_name, @@ -175,8 +176,8 @@ combined as ( sec.student_id, sec.school_year, - case when sec.student_removed_at is not null - then 1 else 0 end as is_removed, + -- case when sec.student_removed_at is not null + -- then 1 else 0 end as is_removed, usr.is_international, usr.country, @@ -236,6 +237,7 @@ final as ( sta.level_id, sta.script_id, sta.course_name, + sta.script_name, sta.level_name, sta.level_type, sta.unit_name, From 55af6645cd48686dcb1a7ee33b800af80c6a1582 Mon Sep 17 00:00:00 2001 From: Jordan Springer Date: Mon, 9 Sep 2024 19:40:54 +0000 Subject: [PATCH 10/10] missing field --- dbt/models/marts/students/dim_student_script_level_activity.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/models/marts/students/dim_student_script_level_activity.sql b/dbt/models/marts/students/dim_student_script_level_activity.sql index 1aa4196a..8ffb1ab7 100644 --- a/dbt/models/marts/students/dim_student_script_level_activity.sql +++ b/dbt/models/marts/students/dim_student_script_level_activity.sql @@ -43,6 +43,7 @@ course_structure as ( course_name, course_id, script_id, + script_name, level_id, level_name, level_type,