Merge branch 'main' into adhoc/ssla

dbt/archive/rosetta/02-run_course_structure.sql

@@ -1,7 +1,3 @@
--- updated: 2024-11-18
--- req: dataops-1055
--- auth: js
-
 CREATE OR REPLACE PROCEDURE public.run_course_structure()
  LANGUAGE plpgsql
 AS $$
@@ -70,28 +66,30 @@ begin
                         	lower(le.properties), 
                         	'project_template_level_name', 
                         	true)                               as project_template_level_name,
-
 			json_extract_path_text(
 				lower(le.properties), 
 				'submittable', 
 				true)                               as submittable,
-
 			case
 				when sl.assessment = 1 
 				then 1 else 0 end                   as assessment,
 
 			case
 				when sc.name like 'devices-20__' 
 						then 'csd'
+
 				when sc.name like '%hello%' 
 						then 'hoc'
+
 				when sc.name like 'microbit%' 
 						then 'csd'
+
 				when json_extract_path_text(
 						lower(sc.properties), 
 						'curriculum_umbrella', 
 						true) = '' 
 						then 'other' 
+
 				else lower(json_extract_path_text(
 						lower(sc.properties), 
 						'curriculum_umbrella', 
@@ -105,6 +103,7 @@ begin
 						stage_number,
 						sl.position)                  as level_script_order,
 
+
 			coalesce(
 				json_extract_path_text(
 						lower(c.properties), 
@@ -158,18 +157,6 @@ begin
 		     case
                 when col.level_group_level_id is not null 
 				then 'Y' else 'N' end                   as is_group_level,
-
-       		nullif(
-				json_extract_path_text(
-					sc.properties, 
-						'content_area', true),'') 	as content_area,
-
-			nullif(
-				replace(replace(replace(
-					json_extract_path_text(
-						sc.properties, 
-						'topic_tags', true)
-						,'[',''),']',''),'"',''),'') as topic_tags, 
 
 			le.updated_at                             	as updated_at
 
@@ -220,4 +207,4 @@ begin
 	      group reader_pii;
 
 end;
-$$
+$$

dbt/dev/admin/run_vestigial_check.sql

@@ -0,0 +1,6 @@
+-- run macro drop_vestigial_relations
+{{ config(
+    tags=["exclude_from_production"]
+) }}
+
+{{ drop_vestigial_relations(dry_run=true) }}

dbt/macros/clean_json_array.sql

@@ -0,0 +1,12 @@
+{% macro clean_json_array(column_name) %}
+    replace(
+        replace(
+            replace(
+                {{ column_name }},
+                '[', ''
+            ),
+            ']', ''
+        ),
+        '"', ''
+    )
+{% endmacro %}

dbt/models/marts/courses/dim_course_structure.sql

@@ -63,6 +63,7 @@ student_course_names as (
     where participant_audience = 'student'
     and course_name not like '%self paced pl%'
     and course_name not in ('hoc', 'other')
+    {# and content_area like '%curriculum' #} -- pending data backfill
 ),
 
 script_names as (
@@ -73,9 +74,18 @@ script_names as (
 combined as (
     select distinct 
         -- courses
+        sc.content_area,
+        sc.course_name,
         ug.unit_group_id                                                as course_id,
         ug.unit_group_name                                              as course_name_full,
-        sc.course_name,
+        -- scripts
+        sc.topic_tags,
+
+
+        sl.script_id,
+        sc.script_name,
+        sc.is_standalone,
+        sc.unit,
 
         --flags
         case 
@@ -124,18 +134,6 @@ combined as (
             then 1 
             else 0 
         end                                                             as is_active_student_course,
-
-        -- surrogate key for level_script_id
-        {{ dbt_utils.generate_surrogate_key(
-            ['lev.level_id', 
-             'sc.script_id']) }}                                        as level_script_id,
-
-
-        -- scripts
-        sl.script_id,
-        sc.script_name,
-        sc.is_standalone,
-        sc.unit,
 
         -- stages
         st.stage_id,
@@ -201,6 +199,11 @@ combined as (
         coalesce(
             ug.participant_audience,
             sc.participant_audience)                                    as participant_audience,
+
+        -- surrogate key for level_script_id
+        {{ dbt_utils.generate_surrogate_key(
+            ['lev.level_id', 
+             'sc.script_id']) }}                                        as level_script_id,
 
         lev.updated_at                                                  as updated_at
 
@@ -233,6 +236,8 @@ combined as (
 
 final as (
     select 
+        content_area,
+        topic_tags,
         course_id,
         course_name_full,	
         course_name,

dbt/models/staging/dashboard/base/base_dashboard__scripts.sql

@@ -6,22 +6,57 @@ source as (
 
 renamed as (
     select
-        id as script_id,
-        name as script_name,
+        id          as script_id,
+        name        as script_name,
         created_at,
         updated_at,
         wrapup_video_id,
         user_id,
         login_required,
-        properties,
-        new_name,
-        family_name,
-        published_state,
-        instruction_type,
-        instructor_audience,
-        participant_audience
-    from source
-)
+        lower(new_name)                 as new_name,
+        lower(family_name)              as family_name,
+        lower(published_state)          as published_state,
+        lower(instruction_type)         as instruction_type,
+        lower(instructor_audience)      as instructor_audience,
+        lower(participant_audience)     as participant_audience,
 
-select * 
-from renamed
+        -- json extraction fields 
+        case 
+            when json_extract_path_text(
+                properties, 
+                'curriculum_umbrella') = ''                         then 'other'
+            else lower(
+                json_extract_path_text(
+                    properties, 
+                    'curriculum_umbrella',
+                true))
+        end                         as course_name,
+
+        json_extract_path_text(
+            properties, 
+            'supported_locales')    as supported_locales,
+
+        json_extract_path_text(
+            properties,
+            'version_year')         as version_year,
+
+        json_extract_path_text(
+            properties,
+            'is_course')            as is_standalone,
+
+        regexp_replace(
+            name,
+            '((-)+\\d{4})',
+            '')                     as unit,
+
+        json_extract_path_text(
+            properties, 
+            'content_area', true)   as content_area,
+
+        json_extract_path_text(
+                properties, 
+                'topic_tags', true) as topic_tags
+    from source )
+
+select *
+from renamed

dbt/models/staging/dashboard/stg_dashboard__scripts.sql

@@ -1,69 +1,41 @@
 with
 scripts as (
-    select * 
+    select 
+        *,
+        {{ clean_json_array('topic_tags') }} as topic_tags_cleaned
     from {{ ref('base_dashboard__scripts') }}
 ),
 
 renamed as (
-    select
-        script_id,
-        lower(script_name)              as script_name,
+    select *,
+        listagg(distinct topic_tags_cleaned) within group(order by topic_tags_cleaned asc) as topic_tags_list
+    from scripts 
+    {{ dbt_utils.group_by(21)}}
+),
+
+final as (
+    select 
+        script_id, 
+        script_name,
         wrapup_video_id,
         user_id,
         login_required,
-        lower(new_name)                 as new_name,
-        lower(family_name)              as family_name,
-        lower(published_state)          as published_state,
-        lower(instruction_type)         as instruction_type,
-        lower(instructor_audience)      as instructor_audience,
-        lower(participant_audience)     as participant_audience,
-
-        -- json extraction fields 
-        case 
-            when json_extract_path_text(
-                properties, 
-                'curriculum_umbrella') = ''                         then 'other'
-            else lower(
-                json_extract_path_text(
-                    properties, 
-                    'curriculum_umbrella',
-                true))
-        end as course_name,
-
-        json_extract_path_text(
-            properties, 
-            'supported_locales')    as supported_locales,
-
-        json_extract_path_text(
-            properties,
-            'version_year')         as version_year,
-
-        json_extract_path_text(
-            properties,
-            'is_course')            as is_standalone,
-
-        regexp_replace(
-            script_name,
-            '((-)+\\d{4})',
-            '')                     as unit,
-
+        new_name,
+        family_name,
+        published_state,
+        instruction_type,
+        instructor_audience,
+        participant_audience,
+        course_name,
+        supported_locales,
+        version_year,
+        is_standalone,
+        unit,
+        nullif(content_area,'')     as content_area,
+        nullif(topic_tags_list,'')  as topic_tags,
         created_at,
         updated_at
-    from scripts )
-
-select 
-    *
-    , case 
-        when course_name in (
-            'csc',
-            'csf', 
-            'csd', 
-            'csa', 
-            'csp', 
-            'ai', 
-            'foundations of cs'
-        )
-        then 1 
-        else 0 
-    end                                                             as is_active_student_course
-from renamed
+    from renamed )
+
+select * 
+from final