diff --git a/hilbert_config/hilbert_cli_config.py b/hilbert_config/hilbert_cli_config.py index 04f0bf7..d0f5dfd 100644 --- a/hilbert_config/hilbert_cli_config.py +++ b/hilbert_config/hilbert_cli_config.py @@ -45,6 +45,29 @@ PEDANTIC = False # NOTE: to treat invalid values/keys as errors? INPUT_DIRNAME = './' # NOTE: base location for external resources _SSH_CONFIG_PATH = None +_HILBERT_STATION = '~/bin/hilbert-station' # NOTE: client-side CLI tool (driver) +_HILBERT_STATION_OPTIONS = '' # -v/-q? -t/-d? +_DRY_RUN_MODE = 0 # 0 - normal mode, 1 => server & client side dry-run mode!, 2 => server-side dry mode (no remote executions!), + + +############################################################### +def add_HILBERT_STATION_OPTIONS(opt): + """Add new option to the global _HILBERT_STATION_OPTIONS string of options to client-side driver""" + global _HILBERT_STATION_OPTIONS + _save = _HILBERT_STATION_OPTIONS + + if _HILBERT_STATION_OPTIONS: + _HILBERT_STATION_OPTIONS = _HILBERT_STATION_OPTIONS + ' ' + str(opt) + else: + _HILBERT_STATION_OPTIONS = str(opt) + + return _save + + +def get_HILBERT_STATION_OPTIONS(): + global _HILBERT_STATION_OPTIONS + return _HILBERT_STATION_OPTIONS + ############################################################### def get_SSH_CONFIG(): @@ -129,6 +152,33 @@ def set_INPUT_DIRNAME(d): return t +############################################################### +def start_dry_run_mode(): + global _DRY_RUN_MODE + + if _DRY_RUN_MODE == 0: + _DRY_RUN_MODE = 1 + add_HILBERT_STATION_OPTIONS('-d') + log.debug("DRY_RUN mode is ON! Will do remote execution in dry-mode") + elif _DRY_RUN_MODE == 1: + # NOTE: dry_run mode twice? + _DRY_RUN_MODE = 2 + log.debug("DRY_RUN mode is ON+ON! No remote executions!") + else: + log.debug("NOTE: DRY_RUN mode is already ON+ON!") + + +def get_DRY_RUN_MODE(): + return _DRY_RUN_MODE + + +def get_NO_REMOTE_EXEC_MODE(): + return (get_DRY_RUN_MODE() >= 2) + +def get_NO_LOCAL_EXEC_MODE(): + return (get_DRY_RUN_MODE() >= 1) + + ############################################################### def start_pedantic_mode(): global PEDANTIC @@ -195,17 +245,27 @@ def pprint(cfg): ############################################################### # timeout=None, -def _execute(_cmd, shell=False, stdout=None, stderr=None): # True??? Try several times? Overall timeout? +def _execute(_cmd, shell=False, stdout=None, stderr=None, dry_run=False): # True??? Try several times? Overall timeout? + """Same as subprocess.call but with logging and possible dry-run mode or pedantic error handling""" + __cmd = ' '.join(_cmd) # stdout = tmp, stderr = open("/dev/null", 'w') # stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) log.debug("Executing shell command: '{}'...".format(__cmd)) + _shell = '' + if shell: + _shell = ' through the shell' + retcode = None try: # with subprocess.Popen(_cmd, shell=shell, stdout=stdout, stderr=stderr) as p: # timeout=timeout, - retcode = subprocess.call(_cmd, shell=shell, stdout=stdout, stderr=stderr) + if dry_run: + print("[Dry-Run-Mode] Execute [{0}]{1}".format(__cmd, _shell)) + retcode = 0 + else: + retcode = subprocess.call(_cmd, shell=shell, stdout=stdout, stderr=stderr) except: log.exception("Could not execute '{0}'! Exception: {1}".format(__cmd, sys.exc_info())) raise @@ -1058,7 +1118,7 @@ def check_script(self, script): _cmd = ["bash", "-n", path] try: # NOTE: Check for valid bash script - retcode = _execute(_cmd) + retcode = _execute(_cmd, dry_run=get_NO_LOCAL_EXEC_MODE()) except: log.exception("Error while running '{}' to check auto-detection script!".format(' '.join(_cmd))) return False # if PEDANTIC: # TODO: add a special switch? @@ -1073,7 +1133,7 @@ def check_script(self, script): _cmd = ["shellcheck", "-s", "bash", path] try: # NOTE: Check for valid bash script - retcode = _execute(_cmd) + retcode = _execute(_cmd, dry_run=get_NO_LOCAL_EXEC_MODE()) except: log.exception("Error while running '{}' to check auto-detection script!".format(' '.join(_cmd))) return False @@ -1213,7 +1273,7 @@ def check(self): return False if PEDANTIC: - if not self.check_ssh_alias(_h, shell=False, timeout=2): + if not self.check_ssh_alias(_h, timeout=2): return False return True @@ -1228,6 +1288,9 @@ def recheck(self): return self.check_ssh_alias(self.get_address()) def rsync(self, source, target, **kwargs): + if get_NO_REMOTE_EXEC_MODE(): + kwargs['dry_run'] = True + log.debug("About to rsync/ssh %s -> %s:%s...", source, str(self.get_address()), target) log.debug("rsync(%s, %s, %s [%s])", self, source, target, str(kwargs)) @@ -1266,6 +1329,9 @@ def rsync(self, source, target, **kwargs): # TODO: check/use SSH/SCP calls! def scp(self, source, target, **kwargs): + if get_NO_REMOTE_EXEC_MODE(): + kwargs['dry_run'] = True + assert self.recheck() _h = self.get_address() # 'jabberwocky' # @@ -1295,6 +1361,9 @@ def scp(self, source, target, **kwargs): return retcode def ssh(self, cmd, **kwargs): + if get_NO_REMOTE_EXEC_MODE(): + kwargs['dry_run'] = True + assert self.recheck() _h = self.get_address() # 'jabberwocky' @@ -1327,6 +1396,9 @@ def check_ssh_alias(cls, _h, **kwargs): """Check for ssh alias""" timeout = kwargs.pop('timeout', 2) + if get_NO_REMOTE_EXEC_MODE(): + kwargs['dry_run'] = True + log.debug("Checking ssh alias: '{0}'...".format(text_type(_h))) try: # client = paramiko.SSHClient() @@ -1518,8 +1590,6 @@ class DockerMachine(BaseRecordValidator): """DockerMachine :: StationPowerOnMethod""" # _DM = 'docker-machine' - _HILBERT_STATION = '~/bin/hilbert-station' # TODO: look at station for this..? - def __init__(self, *args, **kwargs): super(DockerMachine, self).__init__(*args, **kwargs) @@ -1571,9 +1641,9 @@ def start(self): # , action, action_args): _n = self.get_vm_name() # DISPLAY =:0 - _cmd = [self._HILBERT_STATION, 'dm_start', _n] # self._DM + _cmd = [_HILBERT_STATION, _HILBERT_STATION_OPTIONS, 'dm_start', _n] # self._DM try: - _ret = _a.ssh(_cmd, shell=True) + _ret = _a.ssh(_cmd, shell=True) # TODO: check for that shell=True!!!??? except: s = "Could not power-on virtual station {0} (at {1})".format(_n, _a) if not PEDANTIC: @@ -1635,7 +1705,7 @@ def start(self): # , action, action_args): _cmd = [self._WOL, _MAC] # NOTE: avoid IP for now? {"-i", _address, } __cmd = ' '.join(_cmd) try: - retcode = _execute(_cmd, shell=False) + retcode = _execute(_cmd, dry_run=get_NO_LOCAL_EXEC_MODE()) assert retcode is not None if not retcode: log.debug("Command ({}) execution success!".format(__cmd)) @@ -1662,7 +1732,7 @@ def start(self): # , action, action_args): _cmd = [self._WOL, "-i", _address, _MAC] __cmd = ' '.join(_cmd) try: - retcode = _execute(_cmd, shell=False) + retcode = _execute(_cmd, dry_run=get_NO_LOCAL_EXEC_MODE()) assert retcode is not None if not retcode: log.debug("Command ({}) execution success!".format(__cmd)) @@ -1975,8 +2045,6 @@ class Station(BaseRecordValidator): # Wrapper? _client_settings_tag = text_type('client_settings') _type_tag = text_type('type') - _HILBERT_STATION = '~/bin/hilbert-station' - def __init__(self, *args, **kwargs): super(Station, self).__init__(*args, **kwargs) @@ -2009,8 +2077,6 @@ def __init__(self, *args, **kwargs): self._compatible_applications = {} - self._HILBERT_STATION = '~/bin/hilbert-station' - self._profile = None def add_application(self, app_id, app): @@ -2059,7 +2125,7 @@ def get_profile_ref(self): assert _profile_id is not None assert _profile_id != '' - log.debug("Station's profile ID: %s", _profile_id) +# log.debug("Station's profile ID: %s", _profile_id) return _profile_id @@ -2131,7 +2197,7 @@ def shutdown(self): assert isinstance(_a, HostAddress) try: - _ret = _a.ssh([self._HILBERT_STATION, "stop"], shell=False) + _ret = _a.ssh([_HILBERT_STATION, _HILBERT_STATION_OPTIONS, "stop"]) except: s = "Could not stop Hilbert on the station {}".format(_a) if not PEDANTIC: @@ -2145,7 +2211,7 @@ def shutdown(self): return False try: - _ret = _a.ssh([self._HILBERT_STATION, "shutdown", "now"], shell=False) + _ret = _a.ssh([_HILBERT_STATION, _HILBERT_STATION_OPTIONS, "shutdown", "now"]) except: s = "Could not schedule immediate shutdown on the station {}".format(_a) if not PEDANTIC: @@ -2160,7 +2226,7 @@ def shutdown(self): return False try: - _ret = _a.ssh([self._HILBERT_STATION, "shutdown"], shell=False) + _ret = _a.ssh([_HILBERT_STATION, _HILBERT_STATION_OPTIONS, "shutdown"]) except: s = "Could not schedule delayed shutdown on the station {}".format(_a) if not PEDANTIC: @@ -2232,7 +2298,11 @@ def deploy(self): # NOTE: ATM only compose && Application/ServiceIDs == refs to the same docker-compose.yml! # TODO: NOTE: may differ depending on Station::type! - tmp.write('declare -Agr services_and_applications=(\\\n') + # NOTE: the following variables should be set in order to generate a valid config file: + # hilbert_station_services_and_applications + # hilbert_station_profile_services + # hilbert_station_compatible_applications + tmp.write('declare -Agr hilbert_station_services_and_applications=(\\\n') # ss = [] for k in _serviceIDs: s = all_services.get(k, None) # TODO: check compatibility during verification! @@ -2276,7 +2346,6 @@ def deploy(self): # TODO: app.check() app.copy(tmpdir) - else: log.warning('Default application %s is not in the list of compatible apps!', app) @@ -2291,10 +2360,10 @@ def deploy(self): # NOTE: tmp is now generated! try: - # _cmd = ["scp", path, "{0}:/tmp/{1}".format(_a, os.path.basename(path))] # self._HILBERT_STATION, 'deploy' + # _cmd = ["scp", path, "{0}:/tmp/{1}".format(_a, os.path.basename(path))] # _HILBERT_STATION, 'deploy' # _a.scp(path, remote_tmpdir, shell=False) log.debug("About to deploy %s -> %s... (%s)", tmpdir, remote_tmpdir, str(_a.get_address())) - _a.rsync(tmpdir, remote_tmpdir, shell=False) + _a.rsync(tmpdir, remote_tmpdir) # , dry_run=dry_run) except: log.debug("Exception during deployment!") s = "Could not deploy new local settings to {}".format(_a) @@ -2312,11 +2381,15 @@ def deploy(self): finally: log.debug("Temporary Station Configuration File: {}".format(path)) os.umask(saved_umask) - shutil.rmtree(tmpdir) # NOTE: tmpdir is not empty! - _cmd = [self._HILBERT_STATION, "init", remote_tmpdir] + if not get_NO_LOCAL_EXEC_MODE(): + shutil.rmtree(tmpdir) # NOTE: tmpdir is not empty! + else: + print("[Dry-Run-Mode] Keeping temporary location [{0}] that was prepared for deployment to [{1}]".format(tmpdir, str(_a.get_address()))) + + _cmd = [_HILBERT_STATION, _HILBERT_STATION_OPTIONS, "init", remote_tmpdir] try: - _ret = _a.ssh(_cmd, shell=False) + _ret = _a.ssh(_cmd) except: s = "Could not initialize the station using the new configuration file with {}".format(' '.join(_cmd)) if not PEDANTIC: @@ -2334,6 +2407,7 @@ def deploy(self): # TODO: what about other external resources? docker-compose*.yml etc...? # TODO: restart hilbert-station? + # raise NotImplementedError("Cannot deploy local configuration to this station!") return True @@ -2344,7 +2418,7 @@ def app_change(self, app_id): assert isinstance(_a, HostAddress) try: - _ret = _a.ssh([self._HILBERT_STATION, "app_change", app_id], shell=False) + _ret = _a.ssh([_HILBERT_STATION, _HILBERT_STATION_OPTIONS, "app_change", app_id]) except: s = "Could not change top application on the station '{0}' to '{1}'".format(_a, app_id) if not PEDANTIC: @@ -2363,6 +2437,7 @@ def poweron(self): assert _d is not None poweron = _d.get(self._poweron_tag, None) + if poweron is None: log.error("Missing/wrong Power-On Method configuration for this station!") raise Exception("Missing/wrong Power-On Method configuration for this station!") @@ -2691,36 +2766,40 @@ def __init__(self, *args, **kwargs): self._types = {} def validate(self, d): + """String or a sequence of strings""" + +# log.debug("LIST INPUT: [%s], LIST INPUT TYPE: [%s]", d, type(d)) + if d is None: d = self._default_input_data assert self._default_type is not None assert len(self._types) > 0 - _lc = d.lc # NOTE: determine the class of items based on the version and sample data self._type = self._types[self._default_type] assert self._type is not None if (not isinstance(d, (list, dict, tuple, set))) and isinstance(d, string_types): - try: + try: # NOTE: Test single string item: _d = [self._type.parse(StringValidator.parse(d, parent=self))] - self.get_data(_d) + self.set_data(_d) return True except: pass # Not a single string entry... - # list!? - _ret = True + # NOTE: handle a collection (sequence or mapping) of given _type: _d = [] + _ret = True for idx, i in enumerate(d): # What about a string? _v = None try: _v = self._type.parse(i, parent=self) _d.insert(idx, _v) # append? except ConfigurationError as err: + _lc = d.lc _value_error("[%d]" % idx, d, _lc, "Wrong item in the given sequence!") pprint(err) _ret = False @@ -3145,10 +3224,16 @@ def validate(self, d): _ret = False s = _stations[k] + assert s is not None assert isinstance(s, Station) p_ref = s.get_profile_ref() - assert p_ref in _profiles + + if p_ref not in _profiles: + log.error("bad configuration: station '%s' has unknown/invalid profile specification: '%s'", k, p_ref) + return False + p = _profiles[p_ref] + assert p is not None p.add_station(k, s) s.set_profile(p) diff --git a/hilbert_config/subcmdparser.py b/hilbert_config/subcmdparser.py index 939f550..895b352 100644 --- a/hilbert_config/subcmdparser.py +++ b/hilbert_config/subcmdparser.py @@ -313,5 +313,5 @@ def __call__(self, parser, args, values, option_string=None): except: pass - parser.exit(0) + parser.exit() setattr(args, self.dest, values) diff --git a/station/generate_ogl.sh b/station/generate_ogl.sh index 8d28b70..4dda3bf 100755 --- a/station/generate_ogl.sh +++ b/station/generate_ogl.sh @@ -15,8 +15,9 @@ IMG="$U/${I}:${IMAGE_VERSION}" # IMG="$APP" #IMG="$U/$I:$APP" ID=$(docker images | awk '{ print "[" $1 ":" $2 "]" }' | sort | uniq | grep "\[${IMG}\]") if [ -z "$ID" ]; then - echo "ERROR: no such image '${IMG}'" - exit 2 + echo "NOTE: no dummy image '${IMG}', pulling..." + docker pull "${IMG}" || exit $? +# exit 2 fi #shift @@ -34,8 +35,8 @@ docker rm -vf $C 1>&2 || true docker rmi -f --no-prune=false $D 1>&2 || true -R="-it -a stdin -a stdout -a stderr --label is_top_app=0 --ipc=host --net=host --pid=host -v /etc/localtime:/etc/localtime:ro -v /tmp/:/tmp/:rw" -O="--skip-startup-files --no-kill-all-on-exit --quiet --skip-runit" +R="-it -a stdin -a stdout -a stderr --label is_top_app=0 --ipc=host --net=host -v /etc/localtime:/etc/localtime:ro -v /tmp/:/tmp/:rw" +O="/sbin/my_init --skip-startup-files --quiet --skip-runit" ## Create $C conainer out of $IMG and run customization script in it: docker run $R --name $C $IMG $O -- bash -c 'customize.sh' 1>&2 @@ -61,7 +62,10 @@ rm -Rf $G 1>&2 || true ## generate target archive $G: # TODO: --recursion ? ADDEDFILES=/bin/true /lib/x86_64-linux-gnu/libc.so.6 /lib64/ld-linux-x86-64.so.2 /usr/lib/x86_64-linux-gnu/ ? -docker run $R --rm $D $O -- bash -c "tar czvf $G --hard-dereference --dereference $A && chmod a+rw $G" +docker run $R --rm $D $O -- bash -c "tar czvf $G --hard-dereference --dereference $A && chmod a+rw $G" && \ +(echo "Temporary OGL patch is in $G, copying to $PWD..."; cp "$G" .; ) ## post-cleanup: docker rmi -f --no-prune=false $D 1>&2 || true + + diff --git a/tests/data/miniGroupHilbert.yml b/tests/data/miniGroupHilbert.yml new file mode 100644 index 0000000..cd9d8bc --- /dev/null +++ b/tests/data/miniGroupHilbert.yml @@ -0,0 +1,91 @@ +Version: '0.9' +Services: +Profiles: + P1: + services: [] + description: P1 + name: P1 + supported_types: [] + + P2: + services: [] + description: P2 + name: P2 + supported_types: [] +Stations: + S1: + type: hidden + name: S1 + description: S1 + profile: P1 + omd_tag: standalone + address: localhost + hidden: true + + S2: + type: hidden + name: S1 + description: S1 + profile: P2 + omd_tag: standalone + address: localhost + hidden: true + +Groups: + group_include: { include: S1 } + group_include_seq: { include: [S1] } + group_include_seq2: { include: [group_include, group_include_seq] } + + group_exclude: { S1, exclude: S1 } + group_exclude_seq: { S1, exclude: [S1] } + group_exclude_seq2: { S1, exclude: [group_exclude, group_exclude_seq] } + + group_intersectwith: { S1, intersect_with: S1 } + group_intersectwith_seq: { S1, intersect_with: [S1] } + group_intersectwith_seq2: { S1, intersect_with: [group_intersectwith, group_intersectwith_seq] } + + group_spec: { S1, include: group_include, intersect_with: group_intersectwith, exclude: group_exclude } + group_spec_seq: { S1, include: [ group_include ], intersect_with: [group_intersectwith], exclude: [group_exclude] } + group_spec_seq2: { S1, include: [ group_include, group_include_seq ], intersect_with: [group_intersectwith, group_intersectwith_seq], exclude: [group_exclude, group_exclude_seq] } + + # all the following groups are {S1, S2} + test_group_short_union1_12: {S1, S2} + test_group_short_union1_21: {S2, S1} + test_group_short_union2_12: {S1, P2} + test_group_short_union2_21: {S2, P1} + test_group_short_union3: {P1, P2} + + test_group_union1_12: { include: [S1, S2] } + test_group_union1_21: { include: [S2, S1] } + test_group_union2_12: { include: [S1, P2] } + test_group_union2_21: { include: [S2, P1] } + test_group_union3: { include: [P1, P2] } + + test_group_mix_union1_12: {S1, include: S2} + test_group_mix_union1_21: {S2, include: S1} + test_group_mix_union2_12: {S1, include: P2} + test_group_mix_union2_21: {S2, include: P1} + test_group_mix_union3: {P1, include: P2} + + # The following all are {S2} + test_group_exclude_short1: { include: [S1, S2], exclude: S1 } + test_group_exclude_short2: { include: [S1, S2], exclude: P1 } + + test_group_exclude1: { include: [S1, S2], exclude: [S1] } + test_group_exclude2: { include: [S1, S2], exclude: [P1] } + test_group_exclude3: { include: [S1, S2], exclude: [S1, P1] } + + # all the following groups are empty sets + test_group_short_intersect1: {include: S1, intersect_with: S2} + test_group_short_intersect2: {include: S2, intersect_with: S1} + test_group_short_intersect3: {include: S1, intersect_with: P2} + test_group_short_intersect4: {include: S2, intersect_with: P1} + test_group_short_intersect5: {include: P1, intersect_with: P2} + + test_group_intersect1: {include: [S1], intersect_with: [S2]} + test_group_intersect2: {include: [S2], intersect_with: [S1]} + test_group_intersect3: {include: [S1], intersect_with: [P2]} + test_group_intersect4: {include: [S2], intersect_with: [P1]} + test_group_intersect5: {include: [P1], intersect_with: [P2]} + +Applications: diff --git a/tests/data/miniGroupHilbert.yml.data.pickle b/tests/data/miniGroupHilbert.yml.data.pickle new file mode 100644 index 0000000..cce80b4 Binary files /dev/null and b/tests/data/miniGroupHilbert.yml.data.pickle differ diff --git a/tests/data/miniGroupHilbert.yml.pickle b/tests/data/miniGroupHilbert.yml.pickle new file mode 100644 index 0000000..7da5c77 Binary files /dev/null and b/tests/data/miniGroupHilbert.yml.pickle differ diff --git a/tests/data/singleHostHilbert.yml b/tests/data/singleHostHilbert.yml index 01c5750..cfeb8e0 100644 --- a/tests/data/singleHostHilbert.yml +++ b/tests/data/singleHostHilbert.yml @@ -102,21 +102,21 @@ Stations: Groups: # intersectWith is deprecated, use intersect_with instead all5: { station_defaults, main_server, station } # Note: all 3 stations! -# none1: { include: {} } -# none2: {include: {all0}, intersect_with: { all1}, exclude: {all5} } - test1: { include: { main_server, station } } - test2: { main_server, include: { station } } - test3: { include: { main_server}, station } +# none1: { include: [] } +# none2: {include: [all0], intersect_with: [all1], exclude: [all5] } + test1: { include: [ main_server, station ] } + test2: { main_server, include: station } + test3: { include: main_server, station } all0: { all1 } - all1: { standalone, include: {server, simple}, std } - all6: { all1, intersect_with: { all5 } } - sub1: { all0, exclude: { std } } - sub2: { test1, exclude: { std } } - sub3: { test2, exclude: { standalone } } - sub11: { exclude: { std }, all0 } - sub22: { exclude: { std }, test1 } - sub33: { exclude: { standalone }, all5 } - test4: {all6, exclude: {simple}, intersect_with: { server, simple } } + all1: { standalone, include: [server, simple], std } + all6: { all1, intersect_with: all5 } + sub1: { all0, exclude: std } + sub2: { test1, exclude: std } + sub3: { test2, exclude: standalone } + sub11: { exclude: std, all0 } + sub22: { exclude: std, test1 } + sub33: { exclude: standalone, all5 } + test4: {all6, exclude: simple, intersect_with: [ server, simple ] } # a: {a, b, exclude: {a, b}, intersect_with: {a, b}, include: {a, b}} # b: { a} # c: { exclude: {none1} } diff --git a/tests/test_validate.py b/tests/test_validate.py index ffca248..a61588c 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -116,6 +116,9 @@ def test_sample(self, capsys): def test_single(self, capsys): hilbert_validation('singleHostHilbert.yml', 'singleHostHilbert.yml.data.pickle') + def test_groups(self, capsys): + hilbert_validation('miniGroupHilbert.yml', 'miniGroupHilbert.yml.data.pickle') + def test_non_unique_app_service_ids(self, capsys): test_output = None diff --git a/tools/hilbert-station b/tools/hilbert-station index c6d8234..f3f694f 100755 --- a/tools/hilbert-station +++ b/tools/hilbert-station @@ -43,7 +43,7 @@ declare -ir LOCKFD=99 ## PRIVATE flock WRAPPER API function _lock() { flock "-$1" ${LOCKFD}; } -function _no_more_locking() { _lock u; _lock xn && rm -f "${LOCKFILE}"; } +function _no_more_locking() { _lock u; _lock xn && rm -f "${LOCKFILE}"; exec 2>&4 1>&3; } function _prepare_locking() { eval "exec $LOCKFD>\"$LOCKFILE\""; trap _no_more_locking EXIT; } ## PUBLIC flock WRAPPER API @@ -152,7 +152,7 @@ fi ## @fn cmd_usage Show CLI usage help function cmd_usage() { cat << EOF -usage: ${TOOL} [-h] [-p] [-V] [-v | -q] subcommand +usage: ${TOOL} [-h] [-d|-D] [-t|-T] [-V] [-v|-q] subcommand Hilbert - client part for Linux systems @@ -370,7 +370,7 @@ while getopts ":hqtTdDvV" opt; do # NOTE: removed 'p' for now ;; d ) DEBUG "Turning on dry-run mode..." - DRY_RUN="dry-run-mode" + DRY_RUN="Dry-Run-Mode" ;; D ) DEBUG "Turning off dry-run mode..." @@ -440,7 +440,7 @@ function _service_get_type() { function _service_get_field() { local k="$1:$2" - echo "${services_and_applications[$k]}" + echo "${hilbert_station_services_and_applications[$k]}" return $? } @@ -556,18 +556,49 @@ function cmd_install_station_config() { ## @fn cmd_read_configuration function cmd_read_configuration() { if [[ ! -r "${HILBERT_CONFIG_FILE}" ]]; then - ERROR "Could not read configuration from '${HILBERT_CONFIG_FILE}'!" + ERROR "Configuration file '${HILBERT_CONFIG_FILE}' is not readable ($(ls -la ${HILBERT_CONFIG_FILE})!" exit 1 fi - # NOTE: read station configuration: + # NOTE: try to read/check station configuration only once (if it is valid): + declare -p "hilbert_station_services_and_applications" 1>>/dev/null 2>&1 + if [[ $? -eq 0 ]]; then + DEBUG "Configuration file has been loaded already!" + return 0 + fi + + declare -p "hilbert_station_profile_services" 1>>/dev/null 2>&1 + if [[ $? -eq 0 ]]; then + DEBUG "Configuration file has been loaded already!" + return 0 + fi + + declare -p "hilbert_station_profile_services" 1>>/dev/null 2>&1 + if [[ $? -eq 0 ]]; then + DEBUG "Configuration file has been loaded already!" + return 0 + fi + source "${HILBERT_CONFIG_FILE}" # TODO: FIXME: only pass declare ... key="...value..."!? + local _ret=$? - if [[ -v services_and_applications ]]; then - ERROR "Configuration file '${HILBERT_CONFIG_FILE}' does not set 'services_and_applications'!" + declare -p "hilbert_station_services_and_applications" 1>>/dev/null 2>&1 + if [[ $? -ne 0 ]]; then + ERROR "Wrong local configuration file: '${HILBERT_CONFIG_FILE}'! NOTE: it should set at least 'hilbert_station_services_and_applications'!" exit 1 fi - return $? + + declare -p "hilbert_station_profile_services" 1>>/dev/null 2>&1 + if [[ $? -ne 0 ]]; then + DEBUG "Strange local configuration file: '${HILBERT_CONFIG_FILE}': 'hilbert_station_profile_services' is not set!" + fi + + declare -p "hilbert_station_compatible_applications" 1>>/dev/null 2>&1 + if [[ $? -ne 0 ]]; then + DEBUG "Strange local configuration file: '${HILBERT_CONFIG_FILE}': 'hilbert_station_compatible_applications' is not set!" + fi + + return ${_ret} } ## @fn cmd_init @@ -577,7 +608,7 @@ function cmd_init() { if [[ ! -w "/dev/pts/ptmx" ]]; then if [[ -n "${DRY_RUN}" ]]; then - echo "Running: [chmod a+rw /dev/pts/ptmx]..." + echo "[${DRY_RUN}] Running: [chmod a+rw /dev/pts/ptmx]..." else DEBUG "Trying to fix '/dev/pts/ptmx'-permissions:" @@ -613,7 +644,7 @@ function cmd_init() { if [[ -n "${DOCKER_GC}" ]]; then if hash "${DOCKER_GC}" 2>/dev/null; then if [[ -z "${DRY_RUN}" ]]; then - DEBUG "Running: [${DOCKER_GC}]: " + echo "[${DRY_RUN}] Running: [${DOCKER_GC}]: " # ${DOCKER_GC} # NOTE: actually run this in production! else echo "Running: [${DOCKER_GC}]" @@ -671,6 +702,8 @@ function cmd_start_service() { exit 1 fi + cmd_read_configuration + local t t="$(_service_get_type "${arg}")" @@ -703,6 +736,8 @@ function cmd_init_service() { exit 1 fi + cmd_read_configuration + local t t="$(_service_get_type "${arg}")" @@ -737,6 +772,8 @@ function cmd_restart_service() { exit 1 fi + cmd_read_configuration + local t t="$(_service_get_type "${arg}")" @@ -771,6 +808,8 @@ function cmd_stop_service() { exit 1 fi + cmd_read_configuration + local t t="$(_service_get_type "${arg}")" @@ -797,7 +836,7 @@ function cmd_compose_start() { export HILBERT_APPLICATION_ID="$arg" # NOTE: To be set as env. variable APP_ID within container... ## NOTE: Simplify COMPOSE_FILE -> launch! - local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/docker-compose.plain.${HILBERT_APPLICATION_ID}.yaml" + local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/cached_for_${HILBERT_APPLICATION_ID}.${HILBERT_APPLICATION_FILE:-docker-compose.yml}" if [[ -r ${PLAIN_COMPOSE_FILE} ]]; then export COMPOSE_FILE="${PLAIN_COMPOSE_FILE}" else @@ -827,7 +866,7 @@ function cmd_compose_init() { export COMPOSE_FILE="${HILBERT_APPLICATION_FILE:-docker-compose.yml}" # Default value export HILBERT_APPLICATION_ID="$arg" # NOTE: To be set as env. variable APP_ID within container... - local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/docker-compose.plain.${HILBERT_APPLICATION_ID}.yaml" + local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/cached_for_${HILBERT_APPLICATION_ID}.${HILBERT_APPLICATION_FILE:-docker-compose.yml}" DEBUG "Trying to simplify: '${COMPOSE_FILE}' -> '${PLAIN_COMPOSE_FILE}': " cmd_docker_compose config > "${PLAIN_COMPOSE_FILE}" ## NOTE: running config once should be enough... ? @@ -863,7 +902,7 @@ function cmd_compose_restart() { export HILBERT_APPLICATION_ID="$arg" # NOTE: To be set as env. variable APP_ID within container... ## NOTE: Simplify COMPOSE_FILE -> launch! - local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/docker-compose.plain.${HILBERT_APPLICATION_ID}.yaml" + local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/cached_for_${HILBERT_APPLICATION_ID}.${HILBERT_APPLICATION_FILE:-docker-compose.yml}" if [[ -r ${PLAIN_COMPOSE_FILE} ]]; then export COMPOSE_FILE="${PLAIN_COMPOSE_FILE}" else @@ -890,7 +929,7 @@ function cmd_compose_stop() { export HILBERT_APPLICATION_ID="$arg" # NOTE: To be set as env. variable APP_ID within container... ## NOTE: Simplify COMPOSE_FILE -> launch! - local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/docker-compose.plain.${HILBERT_APPLICATION_ID}.yaml" + local PLAIN_COMPOSE_FILE="${HILBERT_CONFIG_DIR}/cached_for_${HILBERT_APPLICATION_ID}.${HILBERT_APPLICATION_FILE:-docker-compose.yml}" if [[ -r ${PLAIN_COMPOSE_FILE} ]]; then export COMPOSE_FILE="${PLAIN_COMPOSE_FILE}" else @@ -931,7 +970,7 @@ function cmd_docker_stop() { function cmd_docker_compose() { if [[ -n "${DRY_RUN}" ]]; then - echo "Running: [${DOCKER_COMPOSE} --skip-hostname-check $*]" + echo "[${DRY_RUN}] Running: [${DOCKER_COMPOSE} --skip-hostname-check $*]" return 0 fi @@ -975,7 +1014,7 @@ function cmd_docker_compose() { function cmd_docker() { if [[ -n "${DRY_RUN}" ]]; then - echo "Running: [${DOCKER} $*]" + echo "[${DRY_RUN}] Running: [${DOCKER} $*]" return 0 fi @@ -1310,7 +1349,12 @@ function cmd_stop() { function cmd_shutdown() { local arg=$@ DEBUG "Shutting this system down... Arguments: [${arg}]" - ${SHUTDOWN} ${arg} || sudo -n -P ${SHUTDOWN} ${arg} + + if [[ -n "${DRY_RUN}" ]]; then + echo "[${DRY_RUN}] Running: [${SHUTDOWN} ${arg} || sudo -n -P ${SHUTDOWN} ${arg}]..." + else + ${SHUTDOWN} ${arg} || sudo -n -P ${SHUTDOWN} ${arg} + fi } ## @fn cmd_subcommand_handle Main CLI parser/handler diff --git a/tools/hilbert.py b/tools/hilbert.py index e16b288..e842626 100755 --- a/tools/hilbert.py +++ b/tools/hilbert.py @@ -56,6 +56,24 @@ # log.exception("Uncaught exception! Type: {0}, Value: {1}, TB: {2}".format(type, value, traceback.format_tb(tb))) ##sys.excepthook = main_exception_handler # Install exception handler + +def set_log_level_options(_ctx): + if 'verbose' in _ctx: + c = _ctx['verbose'] + if c is None: + c = 0 + for i in range(c): + add_HILBERT_STATION_OPTIONS('-v') + + if 'quiet' in _ctx: + c = _ctx['quiet'] + if c is None: + c = 0 + for i in range(c): + add_HILBERT_STATION_OPTIONS('-q') + + + @subcmd('cfg_verify', help='verify the correctness of Hilbert Configuration .YAML file') def cmd_verify(parser, context, args): log.debug("Running '{}'".format('cfg_verify')) @@ -66,9 +84,12 @@ def cmd_verify(parser, context, args): args = parser.parse_args(args) cfg = input_handler(parser, vars(context), args) - assert cfg is not None + if cfg is not None: + print("Input is a valid Hilbert configuration!") + else: + print("Input seems to be invalid Hilbert configuration!") + sys.exit(1) - log.debug("Done") return args @@ -96,8 +117,6 @@ def input_handler(parser, ctx, args): fn = os.path.join('.', 'Hilbert.yml') log.info("Missing input file specification: using default: '{}'!".format(fn)) - - if (fn is not None) and (df is not None): log.error("Input file specification clashes with the input dump specification: specify a single input source!") sys.exit(1) @@ -228,7 +247,6 @@ def cmd_query(parser, context, args): log.info("Writing the configuration into '{}'...".format(od)) pickle_dump(od, obj) - log.debug("Done") return args @@ -261,7 +279,6 @@ def cmd_list_applications(parser, context, args): else: print(yaml_dump(obj)) - log.debug("Done") return args @@ -368,7 +385,6 @@ def cmd_list_stations(parser, context, args): print(']') - log.debug("Done") return args @@ -401,7 +417,6 @@ def cmd_list_profiles(parser, context, args): else: print(yaml_dump(obj)) - log.debug("Done") return args @@ -434,7 +449,6 @@ def cmd_list_groups(parser, context, args): else: print(yaml_dump(obj)) - log.debug("Done") return args @@ -467,14 +481,13 @@ def cmd_list_services(parser, context, args): else: print(yaml_dump(obj)) - log.debug("Done") return args - # NOTE: just a helper ATM def cmd_action(parser, context, args, Action=None, appIdRequired=False): args = parser.parse_args(args) _args = vars(args) + _ctx = vars(context) action = Action if action is None: @@ -506,6 +519,7 @@ def cmd_action(parser, context, args, Action=None, appIdRequired=False): elif 'action_args' in _args: action_args = _args.get('action_args', None) + stations = None log.debug("Validating given StationID: '%s'...", stationId) try: @@ -519,6 +533,7 @@ def cmd_action(parser, context, args, Action=None, appIdRequired=False): if stationId not in stations.get_data(): log.error("Invalid StationID (%s)!", stationId) + print() sys.exit(1) station = stations.get_data()[stationId] @@ -526,6 +541,9 @@ def cmd_action(parser, context, args, Action=None, appIdRequired=False): log.debug("StationID is valid according to the Configuration!") log.debug("Running action: '{0} {1}' on station '{2}'".format(action, str(action_args), stationId)) + + set_log_level_options(_ctx) + try: station.run_action(action, action_args) # NOTE: temporary API for now except: @@ -551,7 +569,6 @@ def cmd_start(parser, context, args): cmd_action(parser, context, args, Action=action, appIdRequired=False) - log.debug("Done") return args @@ -572,7 +589,6 @@ def cmd_stop(parser, context, args): cmd_action(parser, context, args, Action=action, appIdRequired=False) - log.debug("Done") return args @@ -593,7 +609,6 @@ def cmd_cfg_deploy(parser, context, args): cmd_action(parser, context, args, Action=action, appIdRequired=False) - log.debug("Done") return args @@ -615,7 +630,6 @@ def cmd_app_start(parser, context, args): cmd_action(parser, context, args, Action=action, appIdRequired=True) - log.debug("Done") return args @@ -638,7 +652,6 @@ def cmd_app_stop(parser, context, args): cmd_action(parser, context, args, Action=action, appIdRequired=True) - log.debug("Done") return args @@ -660,7 +673,6 @@ def cmd_app_change(parser, context, args): cmd_action(parser, context, args, Action=action, appIdRequired=True) - log.debug("Done") return args @@ -680,7 +692,6 @@ def cmd_run_action(parser, context, args): cmd_action(parser, context, args, Action=None, appIdRequired=False) - log.debug("Done") return args @@ -693,6 +704,26 @@ def __call__(self, *args, **kwargs): start_pedantic_mode() +class RemoteTraceModeAction(argparse._StoreTrueAction): + def __init__(self, *args, **kwargs): + super(RemoteTraceModeAction, self).__init__(*args, **kwargs) + + def __call__(self, *args, **kwargs): + super(RemoteTraceModeAction, self).__call__(*args, **kwargs) + add_HILBERT_STATION_OPTIONS('-t') + + +class CountedDryRunModeAction(argparse._CountAction): + def __init__(self, *args, **kwargs): + super(CountedDryRunModeAction, self).__init__(*args, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + new_count = argparse._ensure_value(namespace, self.dest, 0) + 1 + setattr(namespace, self.dest, new_count) + + start_dry_run_mode() # increase + + def _version(): import platform import dill @@ -707,10 +738,11 @@ def _version(): log.debug("logging version: {}".format(logging.__version__)) log.debug("semantic_version version: {}".format(semantic_version.__version__)) - print("Hilbert Configuration API: {}".format(Hilbert(None).get_api_version())) print("Root Logging Level: {}".format(logging.getLevelName(logging.getLogger().level))) print("PEDANTIC mode: {}".format(get_PEDANTIC())) + print("DRY_RUN mode: {}".format(get_DRY_RUN_MODE())) + print("HILBERT_STATION_OPTIONS: {}".format(get_HILBERT_STATION_OPTIONS())) d = os.environ.get('HILBERT_SERVER_CONFIG_PATH', None) if d is not None: @@ -720,21 +752,21 @@ def _version(): if d is not None: print("HILBERT_SERVER_CONFIG_SSH_PATH: {}".format(d)) -# print("INPUT_DIRNAME: {}".format(get_INPUT_DIRNAME())) - log.debug("Done") + +# print("INPUT_DIRNAME: {}".format(get_INPUT_DIRNAME())) -class ListVersionsAction(argparse.Action): +class ListVersionsAction(argparse._VersionAction): def __init__(self, option_strings, *args, **kwargs): super(ListVersionsAction, self).__init__(option_strings=option_strings, *args, **kwargs) def __call__(self, parser, args, values, option_string=None): + set_log_level_options(vars(args)) +# formatter = parser._get_formatter() +# formatter.add_text(version) +# parser._print_message(formatter.format_help(), _sys.stdout) _version() - parser.exit(status=0) - - -# setattr(args, self.dest, values) - + parser.exit() def main(): handler = SubCommandHandler(use_subcommand_help=True, enable_autocompletion=True, @@ -745,9 +777,17 @@ def main(): default=argparse.SUPPRESS, required=False, help="turn on pedantic mode") + handler.add_argument('-t', '--trace', action=RemoteTraceModeAction, + default=argparse.SUPPRESS, required=False, + help="turn on remote verbose-trace mode") + + handler.add_argument('-d', '--dryrun', action=CountedDryRunModeAction, + help="increase dry-run mode") + handler.add_argument('-V', '--version', action=ListVersionsAction, - nargs=0, default=argparse.SUPPRESS, required=False, type=None, metavar=None, help="show %(prog)s's version and exit") + # nargs=0, default=argparse.SUPPRESS, required=False, type=None, metavar=None, + _argv = sys.argv[1:] @@ -755,10 +795,11 @@ def main(): if len(_argv) == 0: log.debug("No command arguments given => Showing usage help!") _argv = ['-h'] - # handler.print_help() + # handler.print_help() args = handler.run(_argv) - handler.exit(status=0) + handler.exit() + if __name__ == "__main__": main()