From 2048bba5577facda61d82ed20b33c0939931f3c0 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Thu, 26 Sep 2024 13:21:24 -0400 Subject: [PATCH] refactor(dialogs): move execution dialogs to separate files --- .../application_logs_dialog.py} | 0 .../ui/views/dialogs/generate_tests_dialog.py | 80 ++++++++++++ .../profiling_results_dialog.py} | 0 .../ui/views/dialogs/run_profiling_dialog.py | 84 ++++++++++++ testgen/ui/views/dialogs/run_tests_dialog.py | 94 ++++++++++++++ testgen/ui/views/profiling_anomalies.py | 2 +- testgen/ui/views/project_settings.py | 2 +- testgen/ui/views/table_groups.py | 42 +----- testgen/ui/views/test_definitions.py | 2 +- testgen/ui/views/test_results.py | 2 +- testgen/ui/views/test_suites.py | 121 +----------------- 11 files changed, 266 insertions(+), 163 deletions(-) rename testgen/ui/views/{app_log_modal.py => dialogs/application_logs_dialog.py} (100%) create mode 100644 testgen/ui/views/dialogs/generate_tests_dialog.py rename testgen/ui/views/{profiling_modal.py => dialogs/profiling_results_dialog.py} (100%) create mode 100644 testgen/ui/views/dialogs/run_profiling_dialog.py create mode 100644 testgen/ui/views/dialogs/run_tests_dialog.py diff --git a/testgen/ui/views/app_log_modal.py b/testgen/ui/views/dialogs/application_logs_dialog.py similarity index 100% rename from testgen/ui/views/app_log_modal.py rename to testgen/ui/views/dialogs/application_logs_dialog.py diff --git a/testgen/ui/views/dialogs/generate_tests_dialog.py b/testgen/ui/views/dialogs/generate_tests_dialog.py new file mode 100644 index 0000000..43c88ac --- /dev/null +++ b/testgen/ui/views/dialogs/generate_tests_dialog.py @@ -0,0 +1,80 @@ +import time +import pandas as pd +import streamlit as st + +import testgen.ui.services.test_suite_service as test_suite_service +from testgen.commands.run_generate_tests import run_test_gen_queries +from testgen.ui.components import widgets as testgen + +ALL_TYPES_LABEL = "All Test Types" + + +@st.dialog(title="Generate Tests") +def generate_tests_dialog(test_suite: pd.Series) -> None: + test_suite_id = test_suite["id"] + test_suite_name = test_suite["test_suite"] + table_group_id = test_suite["table_groups_id"] + + selected_set = "" + generation_sets = test_suite_service.get_generation_set_choices() + + if generation_sets: + generation_sets.insert(0, ALL_TYPES_LABEL) + + with st.container(): + selected_set = st.selectbox("Generation Set", generation_sets) + if selected_set == ALL_TYPES_LABEL: + selected_set = "" + + test_ct, unlocked_test_ct, unlocked_edits_ct = test_suite_service.get_test_suite_refresh_warning(test_suite_id) + if test_ct: + unlocked_message = "" + if unlocked_edits_ct > 0: + unlocked_message = "Manual changes have been made to auto-generated tests in this test suite that have not been locked. " + elif unlocked_test_ct > 0: + unlocked_message = "Auto-generated tests are present in this test suite that have not been locked. " + + warning_message = f""" + {unlocked_message} + Generating tests now will overwrite unlocked tests subject to auto-generation based on the latest profiling. + \n\n_Auto-generated Tests: {test_ct}, Unlocked: {unlocked_test_ct}, Edited Unlocked: {unlocked_edits_ct}_ + """ + + with st.container(): + st.warning(warning_message, icon=":material/warning:") + if unlocked_edits_ct > 0: + if st.button("Lock Edited Tests"): + if test_suite_service.lock_edited_tests(test_suite_id): + st.info("Edited tests have been successfully locked.") + + with st.container(): + st.markdown(f"Execute test generation for the test suite **{test_suite_name}**?") + + if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:generate-tests-show-cli"): + st.code( + f"testgen run-test-generation --table-group-id {table_group_id} --test-suite-key {test_suite_name}", + language="shellSession", + ) + + button_container = st.empty() + status_container = st.empty() + + test_generation_button = None + with button_container: + _, button_column = st.columns([.75, .25]) + with button_column: + test_generation_button = st.button("Generate Tests", use_container_width=True) + + if test_generation_button: + button_container.empty() + status_container.info("Starting test generation ...") + + try: + run_test_gen_queries(table_group_id, test_suite_name, selected_set) + except Exception as e: + status_container.error(f"Test generation encountered errors: {e!s}.") + + status_container.success(f"Test generation completed for test suite **{test_suite_name}**.") + time.sleep(1) + st.cache_data.clear() + st.rerun() diff --git a/testgen/ui/views/profiling_modal.py b/testgen/ui/views/dialogs/profiling_results_dialog.py similarity index 100% rename from testgen/ui/views/profiling_modal.py rename to testgen/ui/views/dialogs/profiling_results_dialog.py diff --git a/testgen/ui/views/dialogs/run_profiling_dialog.py b/testgen/ui/views/dialogs/run_profiling_dialog.py new file mode 100644 index 0000000..3ba1ab7 --- /dev/null +++ b/testgen/ui/views/dialogs/run_profiling_dialog.py @@ -0,0 +1,84 @@ +import time + +import pandas as pd +import streamlit as st + +import testgen.ui.services.query_service as dq +from testgen.commands.run_profiling_bridge import run_profiling_in_background +from testgen.ui.components import widgets as testgen +from testgen.ui.session import session + +LINK_KEY = "run_profiling_dialog:keys:go-to-runs" +LINK_HREF = "profiling-runs" + + +@st.dialog(title="Run Profiling") +def run_profiling_dialog(project_code: str, table_group: pd.Series | None = None, default_table_group_id: str | None = None) -> None: + if table_group is not None and not table_group.empty: + table_group_id: str = table_group["id"] + table_group_name: str = table_group["table_groups_name"] + else: + table_groups_df = get_table_group_options(project_code) + table_group_id: str = testgen.select( + label="Table Group", + options=table_groups_df, + value_column="id", + display_column="table_groups_name", + default_value=default_table_group_id, + required=True, + ) + table_group_name: str = table_groups_df.loc[table_groups_df["id"] == table_group_id, "table_groups_name"].iloc[0] + testgen.whitespace(1) + + with st.container(): + st.markdown(f"Execute profiling for the table group **{table_group_name}**?") + st.markdown(":material/info: _Profiling will be performed in a background process._") + + if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:run-tests-show-cli"): + st.code(f"testgen run-profile --table-group-id {table_group_id}", language="shellSession") + + button_container = st.empty() + status_container = st.empty() + + with button_container: + _, button_column = st.columns([.85, .15]) + with button_column: + profile_button = st.button("Run Profiling", use_container_width=True, disabled=not table_group_id) + + if profile_button: + button_container.empty() + status_container.info("Starting profiling run ...") + + try: + run_profiling_in_background(table_group_id) + except Exception as e: + status_container.error(f"Profiling run encountered errors: {e!s}.") + + # The second condition is needed for the link to work + if profile_button or st.session_state.get(LINK_KEY): + with status_container.container(): + st.success( + f"Profiling run started for table group **{table_group_name}**." + ) + + if session.current_page != LINK_HREF: + testgen.link( + label="Go to Profiling Runs", + href=LINK_HREF, + params={ "table_group": table_group_id }, + right_icon="chevron_right", + underline=False, + height=40, + key=LINK_KEY, + style="margin-left: auto; border-radius: 4px; border: var(--button-stroked-border); padding: 8px 8px 8px 16px; color: var(--primary-color)", + ) + else: + time.sleep(1) + st.cache_data.clear() + st.rerun() + + +@st.cache_data(show_spinner=False) +def get_table_group_options(project_code: str) -> pd.DataFrame: + schema: str = st.session_state["dbschema"] + return dq.run_table_groups_lookup_query(schema, project_code) diff --git a/testgen/ui/views/dialogs/run_tests_dialog.py b/testgen/ui/views/dialogs/run_tests_dialog.py new file mode 100644 index 0000000..b624fb2 --- /dev/null +++ b/testgen/ui/views/dialogs/run_tests_dialog.py @@ -0,0 +1,94 @@ +import time +import pandas as pd +import streamlit as st + +import testgen.ui.services.database_service as db +from testgen.commands.run_execute_tests import run_execution_steps_in_background +from testgen.ui.components import widgets as testgen +from testgen.ui.session import session + +LINK_KEY = "run_tests_dialog:keys:go-to-runs" +LINK_HREF = "test-runs" + + +@st.dialog(title="Run Tests") +def run_tests_dialog(project_code: str, test_suite: pd.Series | None = None, default_test_suite_id: str | None = None) -> None: + if test_suite is not None and not test_suite.empty: + test_suite_id: str = test_suite["id"] + test_suite_name: str = test_suite["test_suite"] + else: + test_suites_df = get_test_suite_options(project_code) + test_suite_id: str = testgen.select( + label="Test Suite", + options=test_suites_df, + value_column="id", + display_column="test_suite", + default_value=default_test_suite_id, + required=True, + ) + test_suite_name: str = test_suites_df.loc[test_suites_df["id"] == test_suite_id, "test_suite"].iloc[0] + testgen.whitespace(1) + + with st.container(): + st.markdown(f"Run tests for the test suite **{test_suite_name}**?") + st.markdown(":material/info: _Test execution will be performed in a background process._") + + if testgen.expander_toggle(expand_label="Show CLI command", key="run_tests_dialog:keys:show-cli"): + st.code( + f"testgen run-tests --project-key {project_code} --test-suite-key {test_suite['test_suite']}", + language="shellSession" + ) + + button_container = st.empty() + status_container = st.empty() + + run_test_button = None + with button_container: + _, button_column = st.columns([.8, .2]) + with button_column: + run_test_button = st.button("Run Tests", use_container_width=True) + + if run_test_button: + button_container.empty() + status_container.info("Starting test run ...") + + try: + run_execution_steps_in_background(project_code, test_suite_name) + except Exception as e: + status_container.error(f"Test run encountered errors: {e!s}.") + + # The second condition is needed for the link to work + if run_test_button or st.session_state.get(LINK_KEY): + with status_container.container(): + st.success( + f"Test run started for test suite **{test_suite_name}**." + ) + + if session.current_page != LINK_HREF: + testgen.link( + label="Go to Test Runs", + href=LINK_HREF, + params={ "test_suite": test_suite_id }, + right_icon="chevron_right", + underline=False, + height=40, + key=LINK_KEY, + style="margin-left: auto; border-radius: 4px; border: var(--button-stroked-border); padding: 8px 8px 8px 16px; color: var(--primary-color)", + ) + else: + time.sleep(1) + st.cache_data.clear() + st.rerun() + + +@st.cache_data(show_spinner=False) +def get_test_suite_options(project_code: str) -> pd.DataFrame: + schema: str = st.session_state["dbschema"] + sql = f""" + SELECT test_suites.id::VARCHAR(50), + test_suites.test_suite + FROM {schema}.test_suites + WHERE test_suites.project_code = '{project_code}' + ORDER BY test_suites.test_suite + """ + return db.retrieve_data(sql) diff --git a/testgen/ui/views/profiling_anomalies.py b/testgen/ui/views/profiling_anomalies.py index a944ed9..46fab19 100644 --- a/testgen/ui/views/profiling_anomalies.py +++ b/testgen/ui/views/profiling_anomalies.py @@ -12,7 +12,7 @@ from testgen.ui.navigation.page import Page from testgen.ui.services import project_service from testgen.ui.session import session -from testgen.ui.views.profiling_modal import view_profiling_button +from testgen.ui.views.dialogs.profiling_results_dialog import view_profiling_button class ProfilingAnomaliesPage(Page): diff --git a/testgen/ui/views/project_settings.py b/testgen/ui/views/project_settings.py index 603d104..f5ab382 100644 --- a/testgen/ui/views/project_settings.py +++ b/testgen/ui/views/project_settings.py @@ -8,7 +8,7 @@ from testgen.ui.navigation.page import Page from testgen.ui.services import form_service, project_service from testgen.ui.session import session -from testgen.ui.views.app_log_modal import view_log_file +from testgen.ui.views.dialogs.application_logs_dialog import view_log_file class ProjectSettingsPage(Page): diff --git a/testgen/ui/views/table_groups.py b/testgen/ui/views/table_groups.py index 1f82de5..879ab92 100644 --- a/testgen/ui/views/table_groups.py +++ b/testgen/ui/views/table_groups.py @@ -10,12 +10,12 @@ import testgen.ui.services.connection_service as connection_service import testgen.ui.services.form_service as fm import testgen.ui.services.table_group_service as table_group_service -from testgen.commands.run_profiling_bridge import run_profiling_in_background from testgen.ui.components import widgets as testgen from testgen.ui.navigation.page import Page from testgen.ui.services import project_service from testgen.ui.services.string_service import empty_if_null from testgen.ui.session import session +from testgen.ui.views.dialogs.run_profiling_dialog import run_profiling_dialog class TableGroupsPage(Page): @@ -114,7 +114,7 @@ def render(self, connection_id: str, **_kwargs) -> None: testgen.button( type_="stroked", label="Run Profiling", - on_click=partial(run_profiling_dialog, table_group), + on_click=partial(run_profiling_dialog, project_code, table_group), key=f"tablegroups:keys:runprofiling:{table_group['id']}", ) @@ -172,44 +172,6 @@ def delete_table_group_dialog(self, table_group: pd.Series): st.rerun() -@st.dialog(title="Run Profiling") -def run_profiling_dialog(table_group: pd.Series) -> None: - table_group_id = table_group["id"] - - with st.container(): - st.markdown( - f"Execute profiling for the Table Group :green[{table_group['table_groups_name']}]?" - " Profiling will be performed in a background process" - ) - - if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:run-tests-show-cli"): - st.code(f"testgen run-profile --table-group-id {table_group_id}", language="shellSession") - - button_container = st.empty() - status_container = st.empty() - - with button_container: - _, button_column = st.columns([.85, .15]) - with button_column: - profile_button = st.button("Start", use_container_width=True) - - if profile_button: - button_container.empty() - - status_container.info("Executing Profiling...") - - try: - run_profiling_in_background(table_group_id) - except Exception as e: - status_container.empty() - status_container.error(f"Process started with errors: {e!s}.") - - status_container.empty() - status_container.success( - "Process has successfully started. Check 'Data Profiling' item in the menu to see the progress." - ) - - def show_table_group_form(mode, project_code: str, connection: dict, table_group: pd.Series | None = None): connection_id = connection["connection_id"] table_groups_settings_tab, table_groups_preview_tab = st.tabs(["Table Group Settings", "Test"]) diff --git a/testgen/ui/views/test_definitions.py b/testgen/ui/views/test_definitions.py index eec1f7e..baae650 100644 --- a/testgen/ui/views/test_definitions.py +++ b/testgen/ui/views/test_definitions.py @@ -16,7 +16,7 @@ from testgen.ui.services import authentication_service, project_service from testgen.ui.services.string_service import empty_if_null, snake_case_to_title_case from testgen.ui.session import session -from testgen.ui.views.profiling_modal import view_profiling_button +from testgen.ui.views.dialogs.profiling_results_dialog import view_profiling_button LOG = logging.getLogger("testgen") diff --git a/testgen/ui/views/test_results.py b/testgen/ui/views/test_results.py index 8f3a4c9..0358595 100644 --- a/testgen/ui/views/test_results.py +++ b/testgen/ui/views/test_results.py @@ -15,7 +15,7 @@ from testgen.ui.services import authentication_service, project_service from testgen.ui.services.string_service import empty_if_null from testgen.ui.session import session -from testgen.ui.views.profiling_modal import view_profiling_button +from testgen.ui.views.dialogs.profiling_results_dialog import view_profiling_button from testgen.ui.views.test_definitions import show_test_form_by_id ALWAYS_SPIN = False diff --git a/testgen/ui/views/test_suites.py b/testgen/ui/views/test_suites.py index ae0205c..f6c9703 100644 --- a/testgen/ui/views/test_suites.py +++ b/testgen/ui/views/test_suites.py @@ -9,8 +9,6 @@ import testgen.ui.services.form_service as fm import testgen.ui.services.query_service as dq import testgen.ui.services.test_suite_service as test_suite_service -from testgen.commands.run_execute_tests import run_execution_steps_in_background -from testgen.commands.run_generate_tests import run_test_gen_queries from testgen.commands.run_observability_exporter import export_test_results from testgen.common import date_service from testgen.ui.components import widgets as testgen @@ -18,6 +16,8 @@ from testgen.ui.navigation.page import Page from testgen.ui.services.string_service import empty_if_null from testgen.ui.session import session +from testgen.ui.views.dialogs.generate_tests_dialog import generate_tests_dialog +from testgen.ui.views.dialogs.run_tests_dialog import run_tests_dialog from testgen.utils import to_int @@ -327,123 +327,6 @@ def delete_test_suite_dialog(selected_test_suite): st.rerun() -@st.dialog(title="Run Tests") -def run_tests_dialog(project_code, selected_test_suite): - test_suite_key = selected_test_suite["test_suite"] - start_process_button_message = "Start" - - with st.container(): - st.markdown(f"Run tests for the test suite :green[{test_suite_key}]?") - - if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:run-tests-show-cli"): - st.code( - f"testgen run-tests --project-key {project_code} --test-suite-key {selected_test_suite['test_suite']}", - language="shellSession" - ) - - button_container = st.empty() - status_container = st.empty() - - run_test_button = None - with button_container: - _, button_column = st.columns([.85, .15]) - with button_column: - run_test_button = st.button(start_process_button_message, use_container_width=True) - - if run_test_button: - button_container.empty() - - status_container.info(f"Running tests for test suite {test_suite_key}") - - try: - run_execution_steps_in_background(project_code, test_suite_key) - except Exception as e: - status_container.empty() - status_container.error(f"Process started with errors: {e!s}.") - - status_container.empty() - status_container.success( - "Process has successfully started. Check details in menu item 'Data Quality Testing'." - ) - - -@st.dialog(title="Generate Tests") -def generate_tests_dialog(selected_test_suite): - test_suite_id = selected_test_suite["id"] - test_suite_key = selected_test_suite["test_suite"] - table_group_id = selected_test_suite["table_groups_id"] - start_process_button_message = "Start" - - with st.container(): - st.markdown(f"Execute the test generation for test suite :green[{test_suite_key}]?") - - warning_container = st.container() - options_container = st.container() - - if testgen.expander_toggle(expand_label="Show CLI command", key="test_suite:keys:generate-tests-show-cli"): - st.code( - f"testgen run-test-generation --table-group-id {table_group_id} --test-suite-key {test_suite_key}", - language="shellSession", - ) - - button_container = st.empty() - status_container = st.empty() - - test_ct, unlocked_test_ct, unlocked_edits_ct = test_suite_service.get_test_suite_refresh_warning(test_suite_id) - if test_ct: - warning_msg = "" - counts_msg = f"\n\nAuto-Generated Tests: {test_ct}, Unlocked: {unlocked_test_ct}, Edited Unlocked: {unlocked_edits_ct}" - if unlocked_edits_ct > 0: - if unlocked_edits_ct > 1: - - warning_msg = "Manual changes have been made to auto-generated tests in this Test Suite that have not been locked. " - else: - warning_msg = "A manual change has been made to an auto-generated test in this Test Suite that has not been locked. " - elif unlocked_test_ct > 0: - warning_msg = "Auto-generated tests are present in this Test Suite that have not been locked. " - warning_msg = f"{warning_msg}Generating tests now will overwrite unlocked tests subject to auto-generation based on the latest profiling.{counts_msg}" - with warning_container: - st.warning(warning_msg) - if unlocked_edits_ct > 0: - lock_edits_button = st.button("Lock Edited Tests") - if lock_edits_button: - edits_locked = test_suite_service.lock_edited_tests(test_suite_id) - if edits_locked: - st.info("Edited tests have been successfully locked.") - - with options_container: - lst_generation_sets = test_suite_service.get_generation_set_choices() - if lst_generation_sets: - lst_generation_sets.insert(0, "(All Test Types)") - str_generation_set = st.selectbox("Generation Set", lst_generation_sets) - if str_generation_set == "(All Test Types)": - str_generation_set = "" - else: - str_generation_set = "" - - test_generation_button = None - with button_container: - _, button_column = st.columns([.85, .15]) - with button_column: - test_generation_button = st.button(start_process_button_message, use_container_width=True) - - if test_generation_button: - button_container.empty() - - table_group_id = selected_test_suite["table_groups_id"] - test_suite_key = selected_test_suite["test_suite"] - status_container.info("Executing Test Generation...") - - try: - run_test_gen_queries(table_group_id, test_suite_key, str_generation_set) - except Exception as e: - status_container.empty() - status_container.error(f"Process had errors: {e!s}.") - - status_container.empty() - status_container.success("Process has successfully finished.") - - @st.dialog(title="Export to Observability") def observability_export_dialog(selected_test_suite): project_key = selected_test_suite["project_code"]