Merge pull request #7 from DataKitchen/release/2.1.8

Release 2.1.8: Minor fixes and improvements

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data-ops-testgen"
-version = "2.1.4"
+version = "2.1.8"
 description = "DataKitchen Inc. Data Quality Engine"
 urls = { "homepage" = "https://datakitchen.io" }
 authors = [
@@ -242,7 +242,7 @@ omit = ["tests/*", "templates/*"]
 skip_empty=true
 
 [tool.bumpver]
-current_version = "2.1.4"
+current_version = "2.1.8"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "release: {old_version} -> {new_version}"
 commit = true

testgen/commands/run_upgrade_db_config.py

@@ -14,6 +14,7 @@ def _get_params_mapping() -> dict:
         "SCHEMA_NAME": get_tg_schema(),
         "TESTGEN_ADMIN_USER": settings.DATABASE_ADMIN_USER,
         "TESTGEN_ADMIN_PASSWORD": settings.DATABASE_ADMIN_PASSWORD,
+        "OBSERVABILITY_URL": settings.OBSERVABILITY_API_URL,
     }
 
 

testgen/template/dbupgrade/0101_incremental_upgrade.sql

@@ -0,0 +1,15 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1
+        FROM information_schema.columns
+        where table_schema = '{SCHEMA_NAME}'
+        and table_name='projects'
+        AND column_name='observability_api_url'
+    ) THEN
+		alter table {SCHEMA_NAME}.projects add column observability_api_url TEXT DEFAULT '';
+        update {SCHEMA_NAME}.projects set observability_api_url='{OBSERVABILITY_URL}' WHERE observability_api_url = '';
+    END IF;
+END $$;

testgen/ui/app.py

@@ -63,7 +63,7 @@ def render(log_level: int = logging.INFO):
     application.logger.debug(f"location status: {session.current_page} {session.current_page_args}")
 
 
-@st.cache_resource(validate=lambda _: not settings.IS_DEBUG)
+@st.cache_resource(validate=lambda _: not settings.IS_DEBUG, show_spinner=False)
 def get_application(log_level: int = logging.INFO):
     return bootstrap.run(log_level=log_level)
 

testgen/ui/assets/style.css

@@ -100,6 +100,11 @@ div[data-modal-container="true"] > div:first-child > div:first-child {
 button[title="Show password text"] {
     display: none;
 }
+
+.element-container:has(iframe[height="0"][title="streamlit_javascript.streamlit_javascript"]),
+.element-container:has(iframe[height="0"][title="testgen.ui.components.utils.component.testgen"]) {
+    display: none !important;
+}
 /* ... */
 
 /* Dark mode */

testgen/ui/components/frontend/js/components/location.js

@@ -8,7 +8,7 @@ import van from '../van.min.js';
 import { Streamlit } from '../streamlit.js';
 
 const Location = (/** @type Properties */ props) => {
-    Streamlit.setFrameHeight('0px');
+    Streamlit.setFrameHeight('0');
 
     van.derive(() => {
         syncHashToCurrentPage(van.val(props.current_page_code));

testgen/ui/queries/test_definition_queries.py

@@ -110,7 +110,7 @@ def update(schema, test_definition):
                     custom_query = NULLIF($${test_definition["custom_query"]}$$, ''),
                     test_definition_status = NULLIF('{test_definition["test_definition_status"]}', ''),
                     export_to_observability = NULLIF('{test_definition["export_to_observability"]}', ''),
-                    column_name = NULLIF('{test_definition["column_name"]}', ''),
+                    column_name = NULLIF($${test_definition["column_name"]}$$, ''),
                     watch_level = NULLIF('{test_definition["watch_level"]}', ''),
                     project_code = NULLIF('{test_definition["project_code"]}', ''),
                     table_groups_id = '{test_definition["table_groups_id"]}'::UUID,
@@ -122,7 +122,7 @@ def update(schema, test_definition):
 
     sql += f"""     test_type = NULLIF('{test_definition["test_type"]}', ''),
                     test_suite = NULLIF('{test_definition["test_suite"]}', ''),
-                    test_description = NULLIF('{test_definition["test_description"]}', ''),
+                    test_description = NULLIF($${test_definition["test_description"]}$$, ''),
                     test_action = NULLIF('{test_definition["test_action"]}', ''),
                     test_mode = NULLIF('{test_definition["test_mode"]}', ''),
                     lock_refresh = NULLIF('{test_definition["lock_refresh"]}', ''),
@@ -133,22 +133,22 @@ def update(schema, test_definition):
                     check_result = NULLIF('{test_definition["check_result"]}', ''),
                     baseline_ct = NULLIF('{test_definition["baseline_ct"]}', ''),
                     baseline_unique_ct = NULLIF('{test_definition["baseline_unique_ct"]}', ''),
-                    baseline_value = NULLIF('{test_definition["baseline_value"]}', ''),
+                    baseline_value = NULLIF($${test_definition["baseline_value"]}$$, ''),
                     baseline_value_ct = NULLIF('{test_definition["baseline_value_ct"]}', ''),
-                    threshold_value = NULLIF('{test_definition["threshold_value"]}', ''),
+                    threshold_value = NULLIF($${test_definition["threshold_value"]}$$, ''),
                     baseline_sum = NULLIF('{test_definition["baseline_sum"]}', ''),
                     baseline_avg = NULLIF('{test_definition["baseline_avg"]}', ''),
                     baseline_sd = NULLIF('{test_definition["baseline_sd"]}', ''),
-                    subset_condition = NULLIF('{test_definition["subset_condition"]}', ''),
-                    groupby_names = NULLIF('{test_definition["groupby_names"]}', ''),
-                    having_condition = NULLIF('{test_definition["having_condition"]}', ''),
-                    window_date_column = NULLIF('{test_definition["window_date_column"]}', ''),
+                    subset_condition = NULLIF($${test_definition["subset_condition"]}$$, ''),
+                    groupby_names = NULLIF($${test_definition["groupby_names"]}$$, ''),
+                    having_condition = NULLIF($${test_definition["having_condition"]}$$, ''),
+                    window_date_column = NULLIF($${test_definition["window_date_column"]}$$, ''),
                     match_schema_name = NULLIF('{test_definition["match_schema_name"]}', ''),
                     match_table_name = NULLIF('{test_definition["match_table_name"]}', ''),
-                    match_column_names = NULLIF('{test_definition["match_column_names"]}', ''),
-                    match_subset_condition = NULLIF('{test_definition["match_subset_condition"]}', ''),
-                    match_groupby_names = NULLIF('{test_definition["match_groupby_names"]}', ''),
-                    match_having_condition = NULLIF('{test_definition["match_having_condition"]}', ''),
+                    match_column_names = NULLIF($${test_definition["match_column_names"]}$$, ''),
+                    match_subset_condition = NULLIF($${test_definition["match_subset_condition"]}$$, ''),
+                    match_groupby_names = NULLIF($${test_definition["match_groupby_names"]}$$, ''),
+                    match_having_condition = NULLIF($${test_definition["match_having_condition"]}$$, ''),
                     window_days = COALESCE({test_definition["window_days"]}, 0)
                 where
                     id = '{test_definition["id"]}'

testgen/ui/views/connections.py

@@ -37,44 +37,49 @@ def render(self) -> None:
 
         tool_bar = tb.ToolBar(long_slot_count=6, short_slot_count=0, button_slot_count=0, prompt=None)
 
+        enable_table_groups = connection["project_host"] and connection["project_db"] and connection["project_qc_schema"]
+
         show_connection_form(connection, project_code)
 
-        status_container = st.empty()
+        if tool_bar.long_slots[-1].button(
+            f":{'gray' if not enable_table_groups else 'green'}[Table Groups　→]",
+            help="Create or edit Table Groups for the Connection",
+            use_container_width=True,
+        ):
+            st.session_state["connection"] = connection.to_dict()
+
+            session.current_page = "connections/table-groups"
+            session.current_page_args = {"connection_id": connection["connection_id"]}
+            st.experimental_rerun()
+
         create_qc_schema_modal = testgen.Modal("Create QC utility schema", "dk-create-qc-schema-modal", max_width=1100)
 
-        if tool_bar.long_slots[0].button(
+        _, col2 = st.columns([70, 30])
+
+        if col2.button(
             "Test Connection",
             help="Verifies that the connection to the database is working",
             use_container_width=True,
         ):
+            status_container = st.empty()
             verify_connection_works(connection, project_code, status_container)
 
-        if tool_bar.long_slots[-2].button(
-            "Create QC Utility schema",
+        if col2.button(
+            "Create QC Utility schema...",
             help="Creates the required Utility schema and related functions in the target database",
             use_container_width=True,
         ):
             create_qc_schema_modal.open()
 
-        if tool_bar.long_slots[-1].button(
-            "Table Groups →",
-            help="Create or edit Table Groups for the selected Connection",
-            use_container_width=True,
-        ):
-            st.session_state["connection"] = connection.to_dict()
-
-            session.current_page = "connections/table-groups"
-            session.current_page_args = {"connection_id": connection["connection_id"]}
-            st.experimental_rerun()
-
         if create_qc_schema_modal.is_open():
             show_create_qc_schema_modal(create_qc_schema_modal, connection)
 
 
 def show_create_qc_schema_modal(modal, selected_connection):
     with modal.container():
-        with st.form("Create QC Utility Schema", clear_on_submit=False):
-            skip_schema_creation = st.toggle("Skip schema creation (only populate functions)")
+        fm.render_modal_header("Create QC Utility Schema", selected_connection["project_qc_schema"])
+        with st.form(clear_on_submit=False):
+            skip_schema_creation = st.toggle("Skip schema creation -- create utility functions in existing QC Schema")
             skip_granting_privileges = st.toggle("Skip granting privileges")
             db_user = st.text_input(label="Admin db user", max_chars=40, placeholder="Optional Field")
             db_password = st.text_input(

testgen/ui/views/profiling_summary.py

@@ -60,7 +60,7 @@ def render(self) -> None:
                 show_record_detail(dct_selected_rows[0])
                 st.markdown(":orange[Click a button to view profiling outcomes for the selected run.]")
             else:
-                st.markdown(":orange[Select a run to see more actions.]")
+                st.markdown(":orange[Select a run to see more information.]")
 
 
 @st.cache_data(show_spinner=False)
@@ -111,7 +111,7 @@ def open_drill_downs(dct_selected_rows, button_slots):
         dct_selected_row = dct_selected_rows[0]
 
     if button_slots[0].button(
-        ":green[Profiling　→]",
+        f":{'gray' if not dct_selected_rows else 'green'}[Profiling　→]",
         help="Review profiling characteristics for each data column",
         use_container_width=True,
         disabled=not dct_selected_rows,
@@ -122,7 +122,7 @@ def open_drill_downs(dct_selected_rows, button_slots):
         st.experimental_rerun()
 
     if button_slots[1].button(
-        ":green[Anomalies　→]",
+        f":{'gray' if not dct_selected_rows else 'green'}[Anomalies　→]",
         help="Review potential data problems identified in profiling",
         use_container_width=True,
         disabled=not dct_selected_rows,

testgen/ui/views/project_settings.py

@@ -6,7 +6,6 @@
 from testgen.ui.navigation.menu import MenuItem
 from testgen.ui.navigation.page import Page
 from testgen.ui.services import form_service, query_service
-from testgen.ui.services import toolbar_service as tb
 from testgen.ui.session import session
 
 
@@ -28,13 +27,8 @@ def render(self) -> None:
             ],
         )
 
-        tool_bar = tb.ToolBar(long_slot_count=6, short_slot_count=0, button_slot_count=0, prompt=None)
         project = get_current_project(session.project)
 
-        test_observability_export = False
-        with tool_bar.long_slots[0]:
-            test_observability_export = st.button("Test Observability Connection", use_container_width=False)
-
         form_service.render_edit_form(
             "",
             project,
@@ -44,7 +38,8 @@ def render(self) -> None:
             form_unique_key="project-settings",
         )
 
-        if test_observability_export:
+        _, col2 = st.columns([70, 30])
+        if col2.button("Test Observability Connection", use_container_width=False):
             status = st.empty()
             status.info("Testing your connection to DataKitchen Observability...")
             try:

testgen/ui/views/table_groups.py

@@ -94,7 +94,7 @@ def render(self, connection_id: int | None = None) -> None:
         ):
             delete_modal.open()
         if tool_bar.short_slots[4].button(
-            "Test Suites　→",
+            f":{'gray' if disable_buttons else 'green'}[Test Suites　→]",
             help="Create or edit Test Suites for the selected Table Group",
             disabled=disable_buttons,
             use_container_width=True,

testgen/ui/views/test_runs.py

@@ -62,7 +62,7 @@ def render(self) -> None:
             dct_selected_row = dct_selected_rows[0] if dct_selected_rows else None
 
             if tool_bar.short_slots[0].button(
-                ":green[Test Results　→]",
+                f":{'gray' if not dct_selected_row else 'green'}[Test Results　→]",
                 help="Review test results for the selected run",
                 use_container_width=True,
                 disabled=not dct_selected_row,

testgen/ui/views/test_suites.py

@@ -102,7 +102,7 @@ def render(self, connection_id: str | None = None, table_group_id: str | None =
             delete_modal.open()
 
         if tool_bar.short_slots[4].button(
-            ":green[Tests →]",
+            f":{'gray' if disable_buttons else 'green'}[Tests　→]",
             help="View and edit Test Definitions for selected Test Suite",
             disabled=disable_buttons,
             use_container_width=True,