From 3be095df3bfbbeb47a9dba549848525df409ec6e Mon Sep 17 00:00:00 2001 From: earthgecko Date: Tue, 3 May 2022 18:20:17 +0100 Subject: [PATCH] Correct misspelt function file IssueID #4530: namespace.analysed_events - Correct misspelt missing function in v3.0.0 - Handle cluster data in api metric endpoint Added: skyline/functions/metrics_manager/namespace_analysed_events.py Deleted: skyline/functions/metrics_manager/namesapce_analysed_events.py Modified: skyline/webapp/backend.py skyline/webapp/webapp.py IssueID #3890: metrics_manager - sync_cluster_files - Use str for fp_id in cluster data Modified: skyline/webapp/ionosphere_backend.py Linted Modified: skyline/analyzer/algorithms.py --- skyline/analyzer/algorithms.py | 2 +- ...events.py => namespace_analysed_events.py} | 0 skyline/webapp/backend.py | 14 +++ skyline/webapp/ionosphere_backend.py | 6 +- skyline/webapp/webapp.py | 102 ++++++++++++------ 5 files changed, 86 insertions(+), 38 deletions(-) rename skyline/functions/metrics_manager/{namesapce_analysed_events.py => namespace_analysed_events.py} (100%) diff --git a/skyline/analyzer/algorithms.py b/skyline/analyzer/algorithms.py index a42d80b0..8dd076d9 100644 --- a/skyline/analyzer/algorithms.py +++ b/skyline/analyzer/algorithms.py @@ -1569,7 +1569,7 @@ def run_selected_algorithm( # @added 20201127 - Feature #3566: custom_algorithms # Handle if the result is None if result is None: - logger.warn('warning :: algorithms :: %s failed to run on %s' % ( + logger.warning('warning :: algorithms :: %s failed to run on %s' % ( str(custom_algorithm), str(base_name))) else: if custom_consensus == 1: diff --git a/skyline/functions/metrics_manager/namesapce_analysed_events.py b/skyline/functions/metrics_manager/namespace_analysed_events.py similarity index 100% rename from skyline/functions/metrics_manager/namesapce_analysed_events.py rename to skyline/functions/metrics_manager/namespace_analysed_events.py diff --git a/skyline/webapp/backend.py b/skyline/webapp/backend.py index 4211b129..29c96298 100644 --- a/skyline/webapp/backend.py +++ b/skyline/webapp/backend.py @@ -1004,6 +1004,13 @@ def get_cluster_data(api_endpoint, data_required, only_host='all', endpoint_para logger.error('error :: get_cluster_data :: failed to get %s from %s' % ( api_endpoint, str(item))) if r: + # @added 20220503 - Feature #4530: namespace.analysed_events + # Added 404 to handle resource not on cluster node + if r.status_code == 404: + logger.warning('get_cluster_data :: %s from %s responded with status code %s and reason %s' % ( + api_endpoint, str(item), str(r.status_code), str(r.reason))) + return data + if r.status_code != 200: logger.error('error :: get_cluster_data :: %s from %s responded with status code %s and reason %s' % ( api_endpoint, str(item), str(r.status_code), str(r.reason))) @@ -1040,6 +1047,13 @@ def get_cluster_data(api_endpoint, data_required, only_host='all', endpoint_para str(remote_data), str(data_required), str(item[0]))) data.append(remote_data) else: + # @added 20220503 - Feature #4530: namespace.analysed_events + # Added 404 to handle resource not on cluster node + if r.status_code == 404: + logger.warning('get_cluster_data :: %s from %s responded with status code %s and reason %s' % ( + api_endpoint, str(item), str(r.status_code), str(r.reason))) + return data + logger.error('error :: get_cluster_data :: failed to get response from %s on %s' % ( api_endpoint, str(item))) diff --git a/skyline/webapp/ionosphere_backend.py b/skyline/webapp/ionosphere_backend.py index d53931e5..036de440 100644 --- a/skyline/webapp/ionosphere_backend.py +++ b/skyline/webapp/ionosphere_backend.py @@ -7109,7 +7109,6 @@ def expected_features_profiles_dirs(): # logger.error('error :: could not determine fp_dir for fp_id %s' % str(fp_id)) # features_profiles_dirs_error_logged = True fp_dir = None - pass if not fp_dir: try: metric_id = fps[fp_id]['metric_id'] @@ -7119,7 +7118,10 @@ def expected_features_profiles_dirs(): fp_dir = '%s/%s/%s' % ( settings.IONOSPHERE_PROFILES_FOLDER, metric_timeseries_dir, str(timestamp)) - features_profile_dirs_dict[fp_id] = fp_dir + # @modified 20220503 - Feature #3890: metrics_manager - sync_cluster_files + # Use a str for jsonify + # features_profile_dirs_dict[fp_id] = fp_dir + features_profile_dirs_dict[str(fp_id)] = fp_dir added_fps += 1 except: logger.error(traceback.format_exc()) diff --git a/skyline/webapp/webapp.py b/skyline/webapp/webapp.py index a27b2f9e..5cb636e6 100644 --- a/skyline/webapp/webapp.py +++ b/skyline/webapp/webapp.py @@ -3343,6 +3343,13 @@ def api(): if 'metric' in request.args: metric = request.args.get(str('metric'), None) + + # @added 20220503 - Feature #4530: namespace.analysed_events + # Handle cluster_data + logger.info('/api?metric=%s' % str(metric)) + timeseries = [] + raw_series = None + try: # @modified 20200225 - Bug #3266: py3 Redis binary objects not strings # Branch #3262: py3 @@ -3354,13 +3361,13 @@ def api(): # @modified 20220115 - Bug #4374: webapp - handle url encoded chars # Roll back change - breaking existing metrics with colons # metric_name = url_encode_metric_name(metric_name) - raw_series = REDIS_CONN_UNDECODE.get(metric_name) - if not raw_series: - resp = json.dumps( - {'results': 'Error: No metric by that name - try /api?metric=' + settings.FULL_NAMESPACE + 'metric_namespace'}) - return resp, 404 - else: + except Exception as err: + logger.error(traceback.format_exc()) + logger.error('error :: could not get raw data from Redis for %s' % metric) + + if raw_series: + try: unpacker = Unpacker(use_list=False) unpacker.feed(raw_series) # @modified 20201117 - Feature #3824: get_cluster_data @@ -3370,36 +3377,61 @@ def api(): # Replace redefinition of item from line 1338 # timeseries = [item[:2] for item in unpacker] timeseries = [ts_item[:2] for ts_item in unpacker] - # @added 20210416 - remove_full_namespace_prefix = False - if 'remove_full_namespace_prefix' in request.args: - remove_full_namespace_prefix_str = request.args.get('remove_full_namespace_prefix', 'false') - if remove_full_namespace_prefix_str == 'true': - remove_full_namespace_prefix = True - if metric.startswith(settings.FULL_NAMESPACE): - metric = metric.replace(settings.FULL_NAMESPACE, '', 1) - if 'format' in request.args: - format = request.args.get(str('format'), 'pjson') - if format == 'json': - duration = (time.time() - start) - data_dict = { - "status": { - "cluster_data": cluster_data, - "format": format, - "response": 200, - "request_time": duration, - "remove_full_namespace_prefix": remove_full_namespace_prefix - }, - "data": { - "metric": metric, - "timeseries": timeseries - } + except Exception as err: + logger.error(traceback.format_exc()) + logger.error('error :: failed to unpack raw data from Redis for %s' % metric) + + if not timeseries: + if settings.REMOTE_SKYLINE_INSTANCES and cluster_data: + redis_metric_data_uri = 'metric=%s&format=json' % str(metric) + try: + timeseries = get_cluster_data(redis_metric_data_uri, 'timeseries') + except: + logger.error(traceback.format_exc()) + logger.error('error :: Webapp could not get timeseries from the remote Skyline instances') + if timeseries: + logger.info('got timeseries of length %s from the remote Skyline instances' % str(len(timeseries))) + else: + logger.warning('warning :: failed to get timeseries from the remote Skyline instances') + + if not timeseries: + data_dict = {"status": {"cluster_data": cluster_data, "response": 404}} + return jsonify(data_dict), 404 + + resp = json.dumps( + {'results': 'Error: No metric by that name - try /api?metric=' + settings.FULL_NAMESPACE + 'metric_namespace'}) + return resp, 404 + + try: + remove_full_namespace_prefix = False + if 'remove_full_namespace_prefix' in request.args: + remove_full_namespace_prefix_str = request.args.get('remove_full_namespace_prefix', 'false') + if remove_full_namespace_prefix_str == 'true': + remove_full_namespace_prefix = True + if metric.startswith(settings.FULL_NAMESPACE): + metric = metric.replace(settings.FULL_NAMESPACE, '', 1) + if 'format' in request.args: + use_format = request.args.get(str('format'), 'pjson') + if use_format == 'json': + duration = (time.time() - start) + data_dict = { + "status": { + "cluster_data": cluster_data, + "format": use_format, + "response": 200, + "request_time": duration, + "remove_full_namespace_prefix": remove_full_namespace_prefix + }, + "data": { + "metric": metric, + "timeseries": timeseries } - return jsonify(data_dict), 200 - resp = json.dumps({'results': timeseries}) - return resp, 200 - except Exception as e: - error = "Error: " + str(e) + } + return jsonify(data_dict), 200 + resp = json.dumps({'results': timeseries}) + return resp, 200 + except Exception as err: + error = "Error: " + str(err) resp = json.dumps({'results': error}) return resp, 500