From 449e2e56a9bbeed07cdb52cf97fd0e6834b78853 Mon Sep 17 00:00:00 2001 From: Luca Nicosia <66721906+LucaNicosia@users.noreply.github.com> Date: Tue, 24 May 2022 16:10:26 +0200 Subject: [PATCH] new custom asserts GUI page (#73) * new custom asserts GUI page * docs update Signed-off-by: LucaNicosia --- docs/custom-assert.md | 2 + suzieq/gui/stlit/custom-asserts.py | 491 +++++++++++++++++++++++++++++ suzieq/gui/stlit/guiutils.py | 1 + suzieq/gui/stlit/pagecls.py | 10 +- 4 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 suzieq/gui/stlit/custom-asserts.py diff --git a/docs/custom-assert.md b/docs/custom-assert.md index dc0cb72775..6a3ab8659a 100644 --- a/docs/custom-assert.md +++ b/docs/custom-assert.md @@ -186,6 +186,8 @@ If one of those filters is set already in the custom assertion, the value will b aren't valid.
Run `config validate what=asserts` to check why assertions are failing. +**It's also possible to run the custom assertions directly from the GUI using the "Assert" page.** + ## Custom assertion with remote data As consequence of using REST to access data remotely, also custom assertions must be defined in the same location where the REST server is. This means that in the case of a remote server, the assertions which are local to the CLI aren't loaded and the user must access the machine where the REST diff --git a/suzieq/gui/stlit/custom-asserts.py b/suzieq/gui/stlit/custom-asserts.py new file mode 100644 index 0000000000..51528dd1ab --- /dev/null +++ b/suzieq/gui/stlit/custom-asserts.py @@ -0,0 +1,491 @@ +from dataclasses import dataclass +from typing import Dict, List, Tuple + +import altair as alt +import pandas as pd +import streamlit as st +import yaml +from st_aggrid import AgGrid, GridOptionsBuilder +from suzieq.gui.stlit.guiutils import (SuzieqMainPages, gui_get_df, + set_def_aggrid_options) +from suzieq.gui.stlit.pagecls import SqGuiPage +from suzieq.shared.exceptions import SqAssertError, SqPollerConfError +from suzieq.shared.utils import ASSERT_RESULTS +from suzieq.sqobjects import get_sqobject, validate_sq_assertions + + +@dataclass +class CustomAssertSessionState: + '''Session state for Search page''' + page: str = SuzieqMainPages.CUSTOM_ASSERTS.value + assert_name: str = '' + custom_asserts: dict = None + assertion_names: list = None + assert_result_df: pd.DataFrame = None + assert_namespace: str = '' + assert_result: str = '' + + +class CustomAssertPage(SqGuiPage): + """Page to visualize and run custom assertions + """ + _title: str = SuzieqMainPages.CUSTOM_ASSERTS.value + _config_file: str = st.session_state.get('config_file', '') + _state = CustomAssertSessionState() + _table_asserts_df: pd.DataFrame = None + _run_clicked: bool = False + _refresh_clicked: bool = False + _validation_error: str = '' + _name_changed: str = True + _first_time: bool = True + _namespaces: list = [] + + @property + def add_to_menu(self): + return True + + def build(self): + self._get_state_from_url() + self._init_assertions() + self._create_sidebar() + layout = self._create_layout() + self._render(layout) + self._save_page_url() + + def _init_assertions(self): + state = self._state + + if self._first_time: + # the first rendering the page must reset the assert result table + # and the stats to avoid displaying old data + self._reset_assert_result() + self._first_time = False + + if (not state.custom_asserts or not state.assertion_names or + self._table_asserts_df is None): + # read custom assertions + # if the code doesn't enter in the if-statement above, cached + # values are used + assert_specs, error = self._get_assertions() + self._validation_error = error + if error: + state.custom_asserts = {} + state.assertion_names = [] + self._table_asserts_df = None + else: + state.custom_asserts = assert_specs + table_asserts = {'table': [], 'name': []} + asrt_names = [''] + for a_name, specs in assert_specs.items(): + table_asserts['table'].append(specs['table']) + table_asserts['name'].append(a_name) + asrt_names.append(a_name) + + state.assertion_names = asrt_names + self._table_asserts_df = pd.DataFrame\ + .from_dict(table_asserts) \ + .sort_values(by=['table', 'name']).reset_index(drop=True) + + devdf = gui_get_df('device', self._config_file, + columns=['namespace', 'hostname']) + + self._namespaces = [''] + + if not devdf.empty: + self._namespaces.extend( + sorted(devdf.namespace.unique().tolist())) + + def _create_sidebar(self) -> None: + state = self._state + + with st.sidebar: + select_box = st.container() + filter_form = st.form('assert_filter_form') + sidebar_layout = { + 'selectbox': select_box, + 'filter_form': filter_form, + 'sep1': st.container(), + 'refresh_button': st.container(), + 'sep2': st.container(), + 'warning': st.container() + } + + # It's really important to keep the order of the elements as they + # are currenty: + # 1. check if the user clicks on the refresh button to retrieve + # the new list of assertions to show + # 2. check if the user clicked on show or run buttons to update + # the state (and more importantly state.assert_name) + # 3. initialize the select box. + # + # If we place the selectbox before one of the blocks above, the + # following problems will appear: + # - the list of assertion will not be updated when the user + # clicks on the refresh buttons + # - state.assert_name will not be updated, so the selectbox will + # be instable since it uses this value to get the index of the + # element to show + + with sidebar_layout['refresh_button']: + help_msg = 'Get the updated list of assertions' + refr_clicked = st.button('Refresh assertion list', + key='refresh_assert_list', + help=help_msg) + + if refr_clicked: + # when the refresh button is clicked, force to read again + # the custom assertions. To do it, the cache must be + # cleared + # Also the result of the assertion may be invalidalid, so + # reset it as well. + self._reset_asserts_list() + self._reset_assert_result() + self._init_assertions() + + names_and_table = self._get_assert_name_selectbox_values() + + # disable the run button if no assertions to run or if there is no + # data to run the assertion on + disable_button = (len(names_and_table) == 1 or + len(self._namespaces) == 1) + + # update the state + self._sync_state() + + if self._name_changed: + # if the name changes, the assert result and stats must + # be reset, otherwise old data will be shown + # + # Also the namespace and result (sidebar) filters must be + # reset, otherwise the user may accidently run the next + # assertion applying the same filters of the previous assertion + self._reset_assert_result() + self._reset_sidebar_filters() + + with sidebar_layout['filter_form']: + state_ns = state.assert_namespace + state_res = state.assert_result + if not state_ns or state_ns not in self._namespaces: + ns_index = 0 + else: + ns_index = self._namespaces.index(state_ns) + + result_choises = [''] + ASSERT_RESULTS + if not state_res or state_res not in result_choises: + res_index = 0 + else: + res_index = result_choises.index(state_res) + + st.selectbox('namespace', key='assert_namespace', + index=ns_index, options=self._namespaces) + + st.selectbox('result', key='assert_result', + index=res_index, options=result_choises) + + self._run_clicked = (st.form_submit_button('Run')) + + if disable_button: + # force the run button to do nothing + self._run_clicked = False + + found = -1 + if state.assert_name: + for i, item in enumerate(names_and_table): + if state.assert_name == item.split(' ')[0]: + found = i + break + ns_index = found if (state.assert_name and found != -1) else 0 + + with sidebar_layout['selectbox']: + st.selectbox('Assertion name', + names_and_table, + key='custom_assert_name', + index=ns_index) + + with sidebar_layout['sep1']: + st.markdown('---') + + with sidebar_layout['sep2']: + st.markdown('---') + + # display a message informing the user that the page is still an + # alpha version + with sidebar_layout['warning']: + self._alpha_page_warning_message() + + def _get_assert_name_selectbox_values(self) -> List[str]: + """Return a list containing the assertion names and its tables. + The format is ' ()'. + + The white-space can be used to separator since the assertion names + cannot contain a blank. + + Returns: + List[str]: list of assertion names and tables + """ + state = self._state + names_and_table = [''] + for a_name, asrt in state.custom_asserts.items(): + table = asrt['table'] + names_and_table += [f'{a_name} ({table})'] + return sorted(names_and_table) + + def _sync_state(self) -> None: + self._update_state() + self._save_page_url() + + def _update_state(self) -> None: + wsstate = st.session_state + state = self._state + + asrt_name = None + + if 'custom_assert_name' in wsstate: + asrt_name = wsstate.custom_assert_name.split(' ')[0] + + if 'assert_result' in wsstate: + state.assert_result = wsstate.assert_result + + if 'assert_namespace' in wsstate: + state.assert_namespace = wsstate.assert_namespace + + if asrt_name is None or asrt_name == state.assert_name: + self._name_changed = False + else: + self._name_changed = True + state.assert_name = asrt_name + + def _create_layout(self) -> dict: + with st.container(): + specs_container, result_stats = st.columns(2) + result_container = st.container() + return ({ + 'assert_specs': specs_container, + 'result_stats': result_stats, + 'assert_table': result_container, + }) + + def _reset_sidebar_filters(self): + self._state.assert_namespace = '' + self._state.assert_result = '' + + def _reset_asserts_list(self): + self._state.custom_asserts = None + + def _reset_assert_result(self): + self._state.assert_result_df = None + + def _render(self, layout: dict) -> None: + state = self._state + + if self._validation_error: + with st.expander('Assert validation failed', expanded=True): + st.error(self._validation_error) + st.stop() + + if not state.custom_asserts: + st.info('No custom assertions to show') + st.stop() + + asrt_name = state.assert_name + run_result = 0 + with layout['assert_table']: + # run the assertion + exp_label = f'{asrt_name} table' if asrt_name else 'Assert table' + with st.expander(exp_label, expanded=True): + if self._run_clicked: + run_result = self._run_custom_assertion(asrt_name) + else: + if state.assert_result_df is None: + # no assertion run yet + st.info("Click 'Run' button to run the assertion") + else: + # the assert result is cached. Show it + self._draw_df(state.assert_result_df) + + with layout['result_stats']: + # display the assertion stats + exp_label = (f'{asrt_name} stats' if asrt_name + else 'Assertion stats') + with st.expander(exp_label, expanded=True): + if run_result != 0: + st.error('Error during assert execution. No stats to show') + else: + if state.assert_result_df is None: + # no assertion run yet + st.info('Run the assertion to get the stats') + else: + self._display_stats() + + with layout['assert_specs']: + # display assert specs + exp_label = (f'{asrt_name} specs' if asrt_name + else 'Assertion specs') + with st.expander(exp_label, expanded=True): + if (not asrt_name): + # no assertion selected + st.info('Select an assertion to display') + else: + specs = state.custom_asserts.get(asrt_name) + if not specs: + # unknow assertion + st.warning(f'No assertion called {asrt_name}') + else: + yaml_specs = yaml.safe_dump(specs) + st.code(yaml_specs, language='yaml') + + def _get_assertions(self) -> Tuple[Dict, str]: + """Get validated assertions and return errors if any + + Returns: + Tuple[Dict, str]: assertions and validation errors + """ + assertions = {} + error = None + try: + assertions = validate_sq_assertions( + [], config_file=self._config_file) + except (SqAssertError, SqPollerConfError, + ModuleNotFoundError) as e: + error = str(e) + return assertions, error + + def _display_stats(self): + state = self._state + + assert_df = state.assert_result_df + + if assert_df.empty: + st.info('No stats to show') + return + + res_count_df = assert_df.groupby(by='result')['hostname'].count()\ + .reset_index() \ + .rename({'hostname': 'count'}, axis=1) + + count_chart = alt.Chart(res_count_df, title='result count')\ + .mark_bar(tooltip=True) \ + .encode(y='result', x='count:Q', + color=alt.Color( + 'result', + scale=alt.Scale( + domain=[ + 'pass', 'fail', 'skip'], + range=['green', 'red', 'orange'])) + ) + + st.altair_chart(count_chart, use_container_width=True) + + fail_df = assert_df.query('result=="fail"') + fail_df = fail_df \ + .explode(column="assertReason") \ + .fillna({'assertReason': ''}) \ + .reset_index(drop=True) + + if not fail_df.empty: + + reasons_count_df = fail_df.groupby(by='assertReason')['hostname'] \ + .count().reset_index() \ + .rename({'hostname': 'count', + 'assertReason': 'reason'}, axis=1) + + reasons_chart = alt.Chart(reasons_count_df, + title='reasons distribution')\ + .mark_bar(tooltip=True) \ + .encode(y='reason', x='count:Q') + + st.altair_chart(reasons_chart, use_container_width=True) + + def _run_custom_assertion(self, asrt_name: str) -> int: + """Execute and display the custom assertion + + Args: + asrt_name (str): name of the assertion to run and display + + Returns: + int: return an integer which indicates if an error accurred during + the assertion: + - 0: no errors + - 1: error in assertion + """ + state = self._state + if not asrt_name: + st.info("Select an assertion to run") + return 0 + else: + specs = state.custom_asserts.get(asrt_name) + if not specs: + st.error(f'Unknown assertion {asrt_name}') + return 1 + else: + table = specs.get('table') + if not table: + st.error(f'table not found for {asrt_name}') + return 1 + else: + try: + sqobj = get_sqobject(table)( + config_file=self._config_file) + + assert_args = { + 'name': asrt_name, + } + + # add the namespace and result only if set. If not set, + # the custom assertion filters will be used (if any) + + if state.assert_namespace != '': + assert_args.update({ + 'namespace': [state.assert_namespace] + }) + + if state.assert_result: + assert_args.update({'result': state.assert_result}) + + result_df = sqobj.aver(**assert_args) + + if 'error' in result_df: + st.error(result_df['error'][0]) + return 1 + + state.assert_result_df = result_df + + if result_df.empty: + st.info('No data to show') + return 0 + + # explode assert reason + if 'assertReason' in result_df: + result_df = result_df \ + .explode(column="assertReason") \ + .fillna({'assertReason': '-'}) \ + .reset_index(drop=True) + self._draw_df(result_df) + return 0 + except Exception as e: + st.error(str(e)) + state.assert_result_df = None + return 1 + + def _draw_df(self, df) -> AgGrid: + gb = GridOptionsBuilder.from_dataframe(df) + gb.configure_pagination(paginationPageSize=25) + + gb.configure_default_column(floatingFilter=True, selectable=False) + + gb.configure_grid_options(domLayout='normal') + + gridOptions = gb.build() + gridOptions = set_def_aggrid_options(gridOptions) + + retmode = 'FILTERED' + grid_response = AgGrid( + df, + gridOptions=gridOptions, + allow_unsafe_jscode=True, + data_return_mode=retmode, + fit_columns_on_grid_load=True, + theme='streamlit', + ) + + return grid_response diff --git a/suzieq/gui/stlit/guiutils.py b/suzieq/gui/stlit/guiutils.py index d7a3e7f92e..98f48b4f6f 100644 --- a/suzieq/gui/stlit/guiutils.py +++ b/suzieq/gui/stlit/guiutils.py @@ -29,6 +29,7 @@ class SuzieqMainPages(str, Enum): SEARCH = "Search" DEVCONF = "Config" TOPO = "Topology" + CUSTOM_ASSERTS = "Asserts" def pandas_df_to_markdown_table(df: pd.DataFrame) -> str: diff --git a/suzieq/gui/stlit/pagecls.py b/suzieq/gui/stlit/pagecls.py index 8e584109da..a1ebee2741 100644 --- a/suzieq/gui/stlit/pagecls.py +++ b/suzieq/gui/stlit/pagecls.py @@ -18,7 +18,10 @@ class SqGuiPage(ABC, SqPlugin): _state = {} # Set it to a dataclass specific to the page _title: str = None # Initialized by each page _first_time: bool = True - _URL_PARAMS_BLACKLIST = ['get_clicked', 'uniq_clicked', 'tables_obj'] + _URL_PARAMS_BLACKLIST = ['get_clicked', 'uniq_clicked', 'tables_obj', + 'table_asserts_df', 'assertion_names', + 'custom_asserts', 'assert_result_df', + 'tot_assert_lines', 'assert_fail_count'] _LIST_URL_PARAMS = ['columns'] @property @@ -99,3 +102,8 @@ def _save_page_url(self) -> None: for param in self._URL_PARAMS_BLACKLIST: state.pop(param, None) st.experimental_set_query_params(**state) + + def _alpha_page_warning_message(self): + st.warning(""" + This page is still an alpha version + """)