-
Notifications
You must be signed in to change notification settings - Fork 7
/
handle_optimizer.py
253 lines (228 loc) · 9.44 KB
/
handle_optimizer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
import logging
import requests
from ruamel import yaml
import shutil,os
import pk_config
from pk_helper import *
import json,time
DEFAULT_prestr_init = 'm_opt_init_'
DEFAULT_prestr_input = 'm_opt_input_'
DEFAULT_prestr_target = 'm_opt_target_'
DEFAULT_prestr_target_query = 'query_'
DEFAULT_prestr_target_minth = 'minth_'
DEFAULT_prestr_target_maxth = 'maxth_'
m_opt_init_params = dict()
m_opt_variables = list()
m_opt_dummy_advice = dict(valid='False',phase='training',vmnumber=0,errmsg='Optimizer is disabled! (dryrun)',confident=0)
m_opt_accessible = True
dryrun_id = 'optimizer'
"""
Description of the DATA STRUCTURES
----------------------------------
Format of optimizer interface for init, input and target parameters
as they must be specified in the policy description:
data:
constants:
m_opt_init_VAR1: 'value'
...
queries:
m_opt_input_VAR2: 'query expr for VAR2' ==> params specified this way are sent as sample for the optimiser
m_opt_input_VAR3: 'query expr for VAR3'
...
m_opt_target_query_TARGET1: 'query expre for TARGET1' ==> params specified this way are sent
m_opt_target_minth_TARGET1: 5
m_opt_target_maxth_TARGET1: 10
...
m_opt_init_... : these params are sent as initialisation parameter for the optimizer
m_opt_input_... : these params are evaluated and sent as sample in each cycle for the optimizer
m_opt_target_...: these params are are used both in init and sample communication
query_...: to specify the query expression to be evaluated and sent as sample in each cycle
minth_...: to specify the minimum threshold for the target variable to be sent as init param
maxth_...: to specify the maximum threshold for the target variable to be sent as init param
=====================================
The following dict (converted later to YAML) is required
when calling the Optimizer REST API initialization method.
This data structure is built based on the above policy description.
m_opt_init_params = {
'varname1': 'value1'
'varname2': 'value2'
...
input_metrics: [
{ name: 'varname3' }
{ name: 'varname4' }
... ]
target_metrics: [
{ name: 'varname3', min_threshold: 'value5', max_threshold: 'value6' }
{ name: 'varname4', min_threshold: 'value7', max_threshold: 'value8' }
... ] }
=====================================
The following list is required to identify the evaluated variables
that are needed when calling the Optimizer REST API sample method.
This data structure is built based on the above policy description.
m_opt_variables =
[ { lname: 'name of variable in its original form: "m_opt_{input/target_query}_VARNAME"
sname: 'name of variable used towards the Optimizer'
query: 'query string associated to the variable'
}
...
]
=====================================
"""
def reset_variables():
m_opt_init_params.update(dict())
m_opt_variables[:] = []
return
def varname_if_init(varname):
config = pk_config.config()
init_prestr=config.get('optimizer_vars_prefix_init',DEFAULT_prestr_init)
if varname.startswith(init_prestr):
return varname[len(init_prestr):]
else:
return None
def varname_if_input(varname):
config = pk_config.config()
input_prestr=config.get('optimizer_vars_prefix_input',DEFAULT_prestr_input)
if varname.startswith(input_prestr):
return varname[len(input_prestr):]
else:
return None
def check_if_target(varname):
config = pk_config.config()
prestr_target=config.get('optimizer_vars_prefix_target',DEFAULT_prestr_target)
return varname.startswith(prestr_target)
def insert_target_structure(m_opt_init_params,key,value):
log=logging.getLogger('pk_optimizer')
config = pk_config.config()
prestr_target = config.get('optimizer_vars_prefix_target',DEFAULT_prestr_target)
prestr_target_query = prestr_target+config.get('optimizer_vars_prefix_target_query',DEFAULT_prestr_target_query)
varname, fieldname = None, None
if key.startswith(prestr_target_query):
varname=key[len(prestr_target_query):]
fieldname='name'
m_opt_variables.append(dict(lname=key,sname=varname,query=value))
prestr_target_minth = prestr_target+config.get('optimizer_vars_prefix_target_minth',DEFAULT_prestr_target_minth)
if key.startswith(prestr_target_minth):
varname=key[len(prestr_target_minth):]
fieldname='min_threshold'
prestr_target_maxth = prestr_target+config.get('optimizer_vars_prefix_target_maxth',DEFAULT_prestr_target_maxth)
if key.startswith(prestr_target_maxth):
varname=key[len(prestr_target_maxth):]
fieldname='max_threshold'
if varname and fieldname:
log.info('(O) => TARGET: {0}/{1}:{2}'.format(varname,fieldname,value))
for atarget in m_opt_init_params['constants']['target_metrics']:
if atarget['name']==varname:
if fieldname!='name':
atarget[fieldname]=value
return
targetdict = dict()
targetdict[fieldname] = value
targetdict['name'] = varname
m_opt_init_params['constants']['target_metrics'].append(targetdict)
return
def collect_init_params_and_variables(policy):
log=logging.getLogger('pk_optimizer')
config = pk_config.config()
if pk_config.dryrun_get(dryrun_id):
log.info('(O) DRYRUN enabled. Skipping...')
return
reset_variables()
m_opt_init_params['constants'] = dict()
for varname,value in policy.get('data',dict()).get('constants',dict()).items():
retvarname = varname_if_init(varname)
if retvarname:
log.info('(O) => INIT: {0}:{1}'.format(retvarname,value))
m_opt_init_params['constants'][retvarname]=value
m_opt_init_params['constants']['input_metrics']=list()
for varname,query in policy.get('data',dict()).get('queries',dict()).items():
retvarname = varname_if_input(varname)
if retvarname:
log.info('(O) => INPUT: {0}:{1}'.format(retvarname,query))
m_opt_init_params['constants']['input_metrics'].append(dict(name=retvarname))
m_opt_variables.append(dict(lname=varname,sname=retvarname,query=query))
m_opt_init_params['constants']['target_metrics']=list()
for varname,query in policy.get('data',dict()).get('queries',dict()).items():
if check_if_target(varname):
insert_target_structure(m_opt_init_params,varname,query)
for onenode in policy.get('scaling',dict()).get('nodes',[]):
if 'm_opt_advice' in onenode.get('scaling_rule',''):
_,omin,omax = limit_instances(None,
onenode.get('min_instances'),
onenode.get('max_instances'))
m_opt_init_params['constants']['min_vm_number']=omin
m_opt_init_params['constants']['max_vm_number']=omax
log.debug('(O) m_opt_init_params (yaml) => {0}'.format(yaml.dump(m_opt_init_params)))
log.debug('(O) m_opt_variables (yaml) => {0}'.format(yaml.dump(m_opt_variables)))
return
def calling_rest_api_init():
global m_opt_accessible
log=logging.getLogger('pk_optimizer')
config = pk_config.config()
if pk_config.dryrun_get(dryrun_id):
log.info('(O) DRYRUN enabled. Skipping...')
return
url = config.get('optimizer_endpoint')+'/init'
log.debug('(O) Calling optimizer REST API init() method: '+url)
try:
response = requests.post(url, data=yaml.dump(m_opt_init_params))
m_opt_accessible = True
except Exception as e:
m_opt_accessible = False
log.error('(O) Calling optimizer REST API init() method raised exception: ' + str(e))
log.info('(O) WARNING: Optimizer is disabled for the current policy.')
return
log.debug('(O) Response: '+str(response))
return
def generate_sample(userqueries=dict(),sysqueries=dict()):
log=logging.getLogger('pk_optimizer')
# if pk_config.dryrun_get(dryrun_id):
# log.info('(O) DRYRUN enabled. Skipping...')
# return dict()
# if not m_opt_accessible:
# return dict()
log.debug('(O) USRQUERIES: {0}'.format(str(userqueries)))
log.debug('(O) SYSQUERIES: {0}'.format(str(sysqueries)))
sample = dict()
sample['sample']=dict()
sample['sample']['input_metrics']=[]
sample['sample']['target_metrics']=[]
for var in m_opt_variables:
log.debug('(O) => Scanning {0} ...'.format(var['lname']))
onesample=dict()
onesample['name']=var['sname']
onesample['value']=None
for vname,vvalue in userqueries.items():
if vname==var['lname']:
onesample['value']=vvalue
if onesample['value'] is not None:
if check_if_target(var['lname']):
sample['sample']['target_metrics'].append(onesample)
else:
sample['sample']['input_metrics'].append(onesample)
sample['sample']['timestamp']=str(time.time()).split('.')[0]
sample['sample']['vm_number']=max(len(sysqueries.get('m_nodes',[])),1)
log.debug('(O) => Generated sample: '+str(sample))
return sample
def calling_rest_api_sample(sample=dict()):
log=logging.getLogger('pk_optimizer')
config = pk_config.config()
if pk_config.dryrun_get(dryrun_id):
log.info('(O) DRYRUN enabled. Skipping...')
return
if not m_opt_accessible:
return
url = config.get('optimizer_endpoint')+'/sample'
log.debug('(O) Calling optimizer REST API sample() method: '+url)
response = requests.post(url, data=yaml.dump(sample))
log.debug('(O) Response: '+str(response))
return
def calling_rest_api_advice():
log=logging.getLogger('pk_optimizer')
if pk_config.dryrun_get(dryrun_id) or not m_opt_accessible:
return m_opt_dummy_advice
config = pk_config.config()
url = config.get('optimizer_endpoint')+'/advice'
log.debug('(O) Calling optimizer REST API advice() method: '+url)
response = requests.get(url).json()
log.debug('(O) Response: {0}'.format(response))
return response