Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(dashboard): zero line when no data found #1480

Draft
wants to merge 9 commits into
base: develop
Choose a base branch
from
Draft

feat(dashboard): zero line when no data found #1480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
716e0b6
feat: fill dashabords with empty event
soleksy-splunk Nov 4, 2024
46a13dd
chore: wip update errors query
soleksy-splunk Nov 21, 2024
3ec0eae
feat: zero line for overview error timechart
soleksy-splunk Nov 22, 2024
2fe0b2a
chore: refactor zero line for all time charts
soleksy-splunk Nov 24, 2024
74af2dc
test: adjust tests
soleksy-splunk Nov 24, 2024
e2a64e6
test: adjust dashboard unit tests
soleksy-splunk Nov 24, 2024
b88a966
Revert "test: adjust dashboard unit tests"
soleksy-splunk Nov 24, 2024
11095ea
test: fix
soleksy-splunk Nov 24, 2024
278b327
test: fix
soleksy-splunk Nov 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 53 additions & 3 deletions splunk_add_on_ucc_framework/dashboard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,34 @@
"| join _time [search index=_internal source=*{addon_name}* action=events_ingested "
'| timechart sum(n_events) as \\"Number of events\\" ]'
)
errors_count = "index=_internal source=*{addon_name}* log_level IN ({log_lvl}) | timechart count as Errors by exc_l"

errors_count = "index=_internal source=*{addon_name}* log_level IN ({log_lvl}) | timechart count as Errors by exc_l "

zero_line_search_query = (
"| append [ gentimes increment=5m [ makeresults "
"| eval start=strftime( "
'if(\\"${time_token}.earliest$\\"=\\"now\\"'
",now(),"
'if(match(\\"${time_token}.earliest$\\",\\"^\\\\d+-\\\\d+-\\\\d+(T?\\\\d+:\\\\d+:\\\\d+(\\\\.\\\\d{{3}}Z)?)$\\"),'
'strptime(\\"${time_token}.earliest$\\", \\"%Y-%m-%dT%H:%M:%S.%N\\")'
',relative_time(now(), \\"${time_token}.earliest$\\")'
")"
"), "
'\\"%m/%d/%Y:%T\\")'
"| eval end=strftime("
'if(\\"${time_token}.latest$\\"=\\"now\\",'
"now(),"
'if(match(\\"${time_token}.latest$\\",\\"^\\\\d+-\\\\d+-\\\\d+(T?\\\\d+:\\\\d+:\\\\d+(\\\\.\\\\d{{3}}Z)?)$\\"),'
'strptime(\\"${time_token}.latest$\\", \\"%Y-%m-%dT%H:%M:%S.%N\\") '
',relative_time(now(), \\"${time_token}.latest$\\")'
")"
"), "
'\\"%m/%d/%Y:%T\\")'
"| return start end] "
"| eval {value_label} = 0 | fields - endhuman starthuman starttime "
"| rename endtime as _time | head (${basic_query_token}:job.resultCount$==0)]"
)

events_count = (
"index=_internal source=*{addon_name}* action=events_ingested | "
'timechart sum(n_events) as \\"Number of events\\"'
Expand All @@ -81,6 +108,7 @@
'| rename st as \\"Source type\\", Bytes as \\"Data volume\\", events as \\"Number of events\\", '
'sparkvolume as \\"Volume trendline (Bytes)\\", sparkevent as \\"Event trendline\\"'
)

table_source_query = (
"index=_internal source=*license_usage.log type=Usage ({determine_by} IN ({lic_usg_condition})) "
"| stats sparkline(sum(b)) as sparkvolume, sum(b) as Bytes by s "
Expand Down Expand Up @@ -174,6 +202,16 @@ def generate_dashboard_content(
errors_count=errors_count.format(
addon_name=addon_name.lower(), log_lvl=error_panel_log_lvl
),
errors_count_zero_line=zero_line_search_query.format(
value_label="Errors",
basic_query_token="error_count",
time_token="overview_time",
),
data_ingestion_and_events_zero_line=zero_line_search_query.format(
value_label="Number of events",
basic_query_token="data_volume",
time_token="overview_time",
),
events_count=events_count.format(addon_name=addon_name.lower()),
)
)
Expand All @@ -186,10 +224,17 @@ def generate_dashboard_content(
data_ingestion=data_ingestion.format(
lic_usg_condition=lic_usg_condition, determine_by=determine_by
),
errors_count=errors_count.format(
addon_name=addon_name.lower(), log_lvl=error_panel_log_lvl
data_ingestion_volume_zero_line=zero_line_search_query.format(
value_label="Data volume",
basic_query_token="data_volume",
time_token="data_ingestion_time",
),
events_count=events_count.format(addon_name=addon_name.lower()),
data_ingestion_event_count_zero_line=zero_line_search_query.format(
value_label="Number of events",
basic_query_token="data_ingestion_events_count",
time_token="data_ingestion_time",
),
table_sourcetype=table_sourcetype_query.format(
lic_usg_condition=lic_usg_condition,
addon_name=addon_name.lower(),
Expand Down Expand Up @@ -225,6 +270,11 @@ def generate_dashboard_content(
errors_count=errors_count.format(
addon_name=addon_name.lower(), log_lvl=error_panel_log_lvl
),
errors_count_tab_zero_line=zero_line_search_query.format(
value_label="Errors",
basic_query_token="error_count_tab",
time_token="errors_tab_time",
),
errors_list=errors_list_query.format(
addon_name=addon_name.lower(), log_lvl=error_panel_log_lvl
),
Expand Down
30 changes: 28 additions & 2 deletions splunk_add_on_ucc_framework/templates/data_ingestion_tab_definition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
},
"title": "Data volume",
"dataSources": {
"primary": "data_ingestion_data_volume_ds"
"primary": "data_ingestion_data_volume_ds_chain"
}
},
"data_ingestion_events_count_viz": {
Expand All @@ -61,7 +61,7 @@
},
"title": "Number of events",
"dataSources": {
"primary": "data_ingestion_events_count_ds"
"primary": "data_ingestion_events_count_ds_chain"
}
},
"data_ingestion_table_viz": {
Expand Down Expand Up @@ -136,6 +136,19 @@
"type": "ds.search",
"options": {
"query": "{{data_ingestion}}",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
},
"enableSmartSources": true
},
"name": "data_volume"
},
"data_ingestion_data_volume_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "data_ingestion_data_volume_ds",
"query": "{{data_ingestion_volume_zero_line}}",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
Expand All @@ -146,6 +159,19 @@
"type": "ds.search",
"options": {
"query": "{{events_count}}",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
},
"enableSmartSources": true
},
"name": "data_ingestion_events_count"
},
"data_ingestion_events_count_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "data_ingestion_events_count_ds",
"query": "{{data_ingestion_event_count_zero_line}}",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
Expand Down
15 changes: 14 additions & 1 deletion splunk_add_on_ucc_framework/templates/errors_tab_definition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
},
"title": "Errors count",
"dataSources": {
"primary": "errors_tab_errors_count_ds"
"primary": "errors_tab_errors_count_ds_chain"
}
},
"errors_tab_errors_list_viz": {
Expand Down Expand Up @@ -81,6 +81,19 @@
"type": "ds.search",
"options": {
"query": "{{errors_count}}",
"queryParameters": {
"earliest": "$errors_tab_time.earliest$",
"latest": "$errors_tab_time.latest$"
},
"enableSmartSources": true
},
"name": "error_count_tab"
},
"errors_tab_errors_count_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "errors_tab_errors_count_ds",
"query": "{{errors_count_tab_zero_line}}",
"queryParameters": {
"earliest": "$errors_tab_time.earliest$",
"latest": "$errors_tab_time.latest$"
Expand Down
30 changes: 28 additions & 2 deletions splunk_add_on_ucc_framework/templates/overview_definition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
},
"title": "Data ingestion",
"dataSources": {
"primary": "overview_data_volume_ds"
"primary": "overview_data_volume_ds_chain"
}
},
"overview_errors_viz": {
Expand All @@ -74,7 +74,7 @@
},
"title": "Errors",
"dataSources": {
"primary": "overview_errors_count_ds"
"primary": "overview_errors_count_ds_chain"
}
}
},
Expand Down Expand Up @@ -103,6 +103,19 @@
"type": "ds.search",
"options": {
"query": "{{data_ingestion_and_events}}",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
},
"enableSmartSources": true
},
"name": "data_volume"
},
"overview_data_volume_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "overview_data_volume_ds",
"query": "{{data_ingestion_and_events_zero_line}}",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
Expand All @@ -113,6 +126,19 @@
"type": "ds.search",
"options": {
"query": "{{errors_count}}",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
},
"enableSmartSources": true
},
"name": "error_count"
},
"overview_errors_count_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "overview_errors_count_ds",
"query": "{{errors_count_zero_line}}",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
Expand Down
30 changes: 28 additions & 2 deletions tests/unit/expected_results/data_ingestion_tab_definition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
},
"title": "Data volume",
"dataSources": {
"primary": "data_ingestion_data_volume_ds"
"primary": "data_ingestion_data_volume_ds_chain"
}
},
"data_ingestion_events_count_viz": {
Expand All @@ -61,7 +61,7 @@
},
"title": "Number of events",
"dataSources": {
"primary": "data_ingestion_events_count_ds"
"primary": "data_ingestion_events_count_ds_chain"
}
},
"data_ingestion_table_viz": {
Expand Down Expand Up @@ -136,6 +136,19 @@
"type": "ds.search",
"options": {
"query": "index=_internal source=*license_usage.log type=Usage (s IN (example_input_one*,example_input_two*)) | timechart sum(b) as Usage | rename Usage as \"Data volume\"",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
},
"enableSmartSources": true
},
"name": "data_volume"
},
"data_ingestion_data_volume_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "data_ingestion_data_volume_ds",
"query": "| append [ gentimes increment=5m [ makeresults | eval start=strftime( if(\"$data_ingestion_time.earliest$\"=\"now\",now(),if(match(\"$data_ingestion_time.earliest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$data_ingestion_time.earliest$\", \"%Y-%m-%dT%H:%M:%S.%N\"),relative_time(now(), \"$data_ingestion_time.earliest$\"))), \"%m/%d/%Y:%T\")| eval end=strftime(if(\"$data_ingestion_time.latest$\"=\"now\",now(),if(match(\"$data_ingestion_time.latest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$data_ingestion_time.latest$\", \"%Y-%m-%dT%H:%M:%S.%N\") ,relative_time(now(), \"$data_ingestion_time.latest$\"))), \"%m/%d/%Y:%T\")| return start end] | eval Data volume = 0 | fields - endhuman starthuman starttime | rename endtime as _time | head ($data_volume:job.resultCount$==0)]",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
Expand All @@ -146,6 +159,19 @@
"type": "ds.search",
"options": {
"query": "index=_internal source=*splunk_ta_uccexample* action=events_ingested | timechart sum(n_events) as \"Number of events\"",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
},
"enableSmartSources": true
},
"name": "data_ingestion_events_count"
},
"data_ingestion_events_count_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "data_ingestion_events_count_ds",
"query": "| append [ gentimes increment=5m [ makeresults | eval start=strftime( if(\"$data_ingestion_time.earliest$\"=\"now\",now(),if(match(\"$data_ingestion_time.earliest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$data_ingestion_time.earliest$\", \"%Y-%m-%dT%H:%M:%S.%N\"),relative_time(now(), \"$data_ingestion_time.earliest$\"))), \"%m/%d/%Y:%T\")| eval end=strftime(if(\"$data_ingestion_time.latest$\"=\"now\",now(),if(match(\"$data_ingestion_time.latest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$data_ingestion_time.latest$\", \"%Y-%m-%dT%H:%M:%S.%N\") ,relative_time(now(), \"$data_ingestion_time.latest$\"))), \"%m/%d/%Y:%T\")| return start end] | eval Number of events = 0 | fields - endhuman starthuman starttime | rename endtime as _time | head ($data_ingestion_events_count:job.resultCount$==0)]",
"queryParameters": {
"earliest": "$data_ingestion_time.earliest$",
"latest": "$data_ingestion_time.latest$"
Expand Down
17 changes: 15 additions & 2 deletions tests/unit/expected_results/errors_tab_definition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
},
"title": "Errors count",
"dataSources": {
"primary": "errors_tab_errors_count_ds"
"primary": "errors_tab_errors_count_ds_chain"
}
},
"errors_tab_errors_list_viz": {
Expand Down Expand Up @@ -80,7 +80,20 @@
"errors_tab_errors_count_ds": {
"type": "ds.search",
"options": {
"query": "index=_internal source=*splunk_ta_uccexample* log_level IN (ERROR, CRITICAL) | timechart count as Errors by exc_l",
"query": "index=_internal source=*splunk_ta_uccexample* log_level IN (ERROR, CRITICAL) | timechart count as Errors by exc_l ",
"queryParameters": {
"earliest": "$errors_tab_time.earliest$",
"latest": "$errors_tab_time.latest$"
},
"enableSmartSources": true
},
"name": "error_count_tab"
},
"errors_tab_errors_count_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "errors_tab_errors_count_ds",
"query": "| append [ gentimes increment=5m [ makeresults | eval start=strftime( if(\"$errors_tab_time.earliest$\"=\"now\",now(),if(match(\"$errors_tab_time.earliest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$errors_tab_time.earliest$\", \"%Y-%m-%dT%H:%M:%S.%N\"),relative_time(now(), \"$errors_tab_time.earliest$\"))), \"%m/%d/%Y:%T\")| eval end=strftime(if(\"$errors_tab_time.latest$\"=\"now\",now(),if(match(\"$errors_tab_time.latest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$errors_tab_time.latest$\", \"%Y-%m-%dT%H:%M:%S.%N\") ,relative_time(now(), \"$errors_tab_time.latest$\"))), \"%m/%d/%Y:%T\")| return start end] | eval Errors = 0 | fields - endhuman starthuman starttime | rename endtime as _time | head ($error_count_tab:job.resultCount$==0)]",
"queryParameters": {
"earliest": "$errors_tab_time.earliest$",
"latest": "$errors_tab_time.latest$"
Expand Down
32 changes: 29 additions & 3 deletions tests/unit/expected_results/overview_definition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
},
"title": "Data ingestion",
"dataSources": {
"primary": "overview_data_volume_ds"
"primary": "overview_data_volume_ds_chain"
}
},
"overview_errors_viz": {
Expand All @@ -74,7 +74,7 @@
},
"title": "Errors",
"dataSources": {
"primary": "overview_errors_count_ds"
"primary": "overview_errors_count_ds_chain"
}
}
},
Expand Down Expand Up @@ -103,6 +103,19 @@
"type": "ds.search",
"options": {
"query": "index=_internal source=*license_usage.log type=Usage (s IN (example_input_one*,example_input_two*)) | timechart sum(b) as Usage | rename Usage as \"Data volume\" | join _time [search index=_internal source=*splunk_ta_uccexample* action=events_ingested | timechart sum(n_events) as \"Number of events\" ]",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
},
"enableSmartSources": true
},
"name": "data_volume"
},
"overview_data_volume_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "overview_data_volume_ds",
"query": "| append [ gentimes increment=5m [ makeresults | eval start=strftime( if(\"$overview_time.earliest$\"=\"now\",now(),if(match(\"$overview_time.earliest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$overview_time.earliest$\", \"%Y-%m-%dT%H:%M:%S.%N\"),relative_time(now(), \"$overview_time.earliest$\"))), \"%m/%d/%Y:%T\")| eval end=strftime(if(\"$overview_time.latest$\"=\"now\",now(),if(match(\"$overview_time.latest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$overview_time.latest$\", \"%Y-%m-%dT%H:%M:%S.%N\") ,relative_time(now(), \"$overview_time.latest$\"))), \"%m/%d/%Y:%T\")| return start end] | eval Number of events = 0 | fields - endhuman starthuman starttime | rename endtime as _time | head ($data_volume:job.resultCount$==0)]",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
Expand All @@ -112,7 +125,20 @@
"overview_errors_count_ds": {
"type": "ds.search",
"options": {
"query": "index=_internal source=*splunk_ta_uccexample* log_level IN (ERROR, CRITICAL) | timechart count as Errors by exc_l",
"query": "index=_internal source=*splunk_ta_uccexample* log_level IN (ERROR, CRITICAL) | timechart count as Errors by exc_l ",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
},
"enableSmartSources": true
},
"name": "error_count"
},
"overview_errors_count_ds_chain": {
"type": "ds.chain",
"options": {
"extend": "overview_errors_count_ds",
"query": "| append [ gentimes increment=5m [ makeresults | eval start=strftime( if(\"$overview_time.earliest$\"=\"now\",now(),if(match(\"$overview_time.earliest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$overview_time.earliest$\", \"%Y-%m-%dT%H:%M:%S.%N\"),relative_time(now(), \"$overview_time.earliest$\"))), \"%m/%d/%Y:%T\")| eval end=strftime(if(\"$overview_time.latest$\"=\"now\",now(),if(match(\"$overview_time.latest$\",\"^\\d+-\\d+-\\d+(T?\\d+:\\d+:\\d+(\\.\\d{3}Z)?)$\"),strptime(\"$overview_time.latest$\", \"%Y-%m-%dT%H:%M:%S.%N\") ,relative_time(now(), \"$overview_time.latest$\"))), \"%m/%d/%Y:%T\")| return start end] | eval Errors = 0 | fields - endhuman starthuman starttime | rename endtime as _time | head ($error_count:job.resultCount$==0)]",
"queryParameters": {
"earliest": "$overview_time.earliest$",
"latest": "$overview_time.latest$"
Expand Down
Loading
Loading