From 6335901e7fcaf52526d2910d9415d35ef5271de3 Mon Sep 17 00:00:00 2001 From: Nan Liu Date: Mon, 9 Jan 2017 13:29:25 -0800 Subject: [PATCH] Add debug log instructions for Snap. --- docs/SNAPTEL.md | 109 +++++++++++++++++---------------- docs/SNAPTELD.md | 154 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 165 insertions(+), 98 deletions(-) diff --git a/docs/SNAPTEL.md b/docs/SNAPTEL.md index 7b528e155..176d79701 100644 --- a/docs/SNAPTEL.md +++ b/docs/SNAPTEL.md @@ -21,19 +21,23 @@ limitations under the License. A powerful telemetry framework ## Usage +The `snap-telemetry` package installs `snaptel` in `/usr/local/sbin/snaptel`, and `snapteld` in `/usr/local/sbin/snapteld`. Either ensure `/usr/local/bin:/usr/local/sbin` is in your path, or use fully qualified filepath to the `snaptel`/`snapteld` binary: + ``` -$ $SNAP_PATH/bin/snaptel [global options] command [command options] [arguments...] +$ snaptel [global options] command [command options] [arguments...] ``` + ### Global Options ``` --url, -u 'http://localhost:8181' Sets the URL to use [$SNAP_URL] --insecure Ignore certificate errors when Snap's API is running HTTPS --api-version, -a 'v1' The Snap API version ---password, -p Password for REST API authentication ---config, -c Path to a config file [$SNAPTEL_CONFIG_PATH] +--password, -p Password for REST API authentication +--config, -c Path to a config file [$SNAPTEL_CONFIG_PATH] --help, -h show help --version, -v print the version ``` + ### Commands ``` metric @@ -41,54 +45,57 @@ plugin task help, h Shows a list of commands or help for one command ``` + ### Command Options + #### task ``` -$ $SNAP_PATH/bin/snaptel task command [command options] [arguments...] +$ snaptel task command [command options] [arguments...] ``` ``` create There are two ways to create a task. - 1) Use a task manifest with [--task-manifest, t] - 2) Provide a workflow manifest and schedule details [--workflow-manifest, -w] - - --task-manifest, -t File path for task manifest to use for task creation. - --workflow-manifest, -w File path for workflow manifest to use for task creation - --interval, -i Interval for the task schedule [ex (simple schedule): 250ms, 1s, 30m (cron schedule): "0 * * * * *"] - --start-date Start date for the task schedule [defaults to today] - --start-time Start time for the task schedule [defaults to now] - --stop-date Stop date for the task schedule [defaults to today] - --stop-time Start time for the task schedule [defaults to now] - --name, -n Optional requirement for giving task names - --duration, -d The amount of time to run the task [appends to start or creates a start time before a stop] - --no-start Do not start task on creation [normally started on creation] - - * Note: Start and stop date/time are optional. -list list -start start -stop stop -remove remove -export export -watch watch -enable enable -help, h Shows a list of commands or help for one command + 1) Use a task manifest with [--task-manifest, t] + 2) Provide a workflow manifest and schedule details [--workflow-manifest, -w] + + --task-manifest value, -t value File path for task manifest to use for task creation. + --workflow-manifest value, -w value File path for workflow manifest to use for task creation + --interval value, -i value Interval for the task schedule [ex (simple schedule): 250ms, 1s, 30m (cron schedule): "0 * * * * *"] + --start-date value Start date for the task schedule [defaults to today] + --start-time value Start time for the task schedule [defaults to now] + --stop-date value Stop date for the task schedule [defaults to today] + --stop-time value Start time for the task schedule [defaults to now] + --name value, -n value Optional requirement for giving task names + --duration value, -d value The amount of time to run the task [appends to start or creates a start time before a stop] + --no-start Do not start task on creation [normally started on creation] + --deadline value The deadline for the task to be killed after started if the task runs too long (All tasks default to 5s) + --max-failures value The number of consecutive failures before Snap disables the task + + * Note: Start and stop date/time are optional. +list list +start start +stop stop +remove remove +export export +watch watch +enable enable +help, h Shows a list of commands or help for one command ``` + #### plugin ``` -$ $SNAP_PATH/bin/snaptel plugin command [command options] [arguments...] +$ snaptel plugin command [command options] [arguments...] ``` ``` -load load - --plugin-asc, -a The armored detached plugin signature file (.asc) -unload unload -t -n -v - --plugin-type, -t The plugin type - --plugin-name, -n The plugin name - --plugin-version, -v '0' The plugin version -list list -help, h Shows a list of commands or help for one command +load load +unload unload +swap swap :: or swap -t -n -v +list list +help, h Shows a list of commands or help for one command ``` + #### metric ``` -$ $SNAP_PATH/bin/snaptel metric command [command options] [arguments...] +$ snaptel metric command [command options] [arguments...] ``` ``` list list @@ -103,7 +110,7 @@ Example Usage In one terminal window, run snapteld (log level is set to 1 and signing is turned off for this example): ``` -$ $SNAP_PATH/bin/snapteld -l 1 -t 0 +$ snapteld --log-level 1 --log-path '' --plugin-trust 0 ``` prepare a task manifest file, for example, task.json with following content: @@ -155,10 +162,10 @@ prepare a workflow manifest file, for example, workflow.json with the following "/intel/mock/foo": {} }, "config": { - "/intel/mock/foo": { - "password": "testval" + "/intel/mock/foo": { + "password": "testval" } - }, + }, "process": [], "publish": [ { @@ -184,14 +191,14 @@ and then: 8. unload the plugins ``` -$ $SNAP_PATH/bin/snaptel plugin load $SNAP_PATH/plugin/snap-plugin-collector-mock1 -$ $SNAP_PATH/bin/snaptel plugin load $SNAP_PATH/plugin/snap-plugin-processor-passthru -$ $SNAP_PATH/bin/snaptel plugin load $SNAP_PATH/plugin/snap-plugin-publisher-mock-file -$ $SNAP_PATH/bin/snaptel plugin list -$ $SNAP_PATH/bin/snaptel task create -t mock-file.json -$ $SNAP_PATH/bin/snaptel task create -w workflow.json -i 1s -d 10s -$ $SNAP_PATH/bin/snaptel task list -$ $SNAP_PATH/bin/snaptel plugin unload -t collector -n mock -v -$ $SNAP_PATH/bin/snaptel plugin unload -t processor -n passthru -v -$ $SNAP_PATH/bin/snaptel plugin unload -t publisher -n publisher -v +$ snaptel plugin load /opt/snap/plugins/snap-plugin-collector-mock1 +$ snaptel plugin load /opt/snap/plugins/snap-plugin-processor-passthru +$ snaptel plugin load /opt/snap/plugins/snap-plugin-publisher-mock-file +$ snaptel plugin list +$ snaptel task create -t mock-file.json +$ snaptel task create -w workflow.json -i 1s -d 10s +$ snaptel task list +$ snaptel plugin unload collector mock +$ snaptel plugin unload processor passthru +$ snaptel plugin unload publisher publisher ``` diff --git a/docs/SNAPTELD.md b/docs/SNAPTELD.md index 56587cb6f..08b5ec953 100644 --- a/docs/SNAPTELD.md +++ b/docs/SNAPTELD.md @@ -21,74 +21,134 @@ limitations under the License. The Snap daemon/agent (snapteld) is a modular application that consists of a control module, a scheduler module, and a REST API. The control module is responsible for loading and unloading plugins, managing loaded plugins, and maintaining an available pool of running plugins for running tasks. The scheduler module is responsible for running the workflows in created tasks per the schedule stated. The REST API provides an interface for loading and unloading plugins, creating and removing tasks, starting and stopping tasks, and listing metrics available for collection. ## Usage +The `snap-telemetry` package installs `snapteld` in `/usr/local/sbin/snapteld`. Either ensure `/usr/local/sbin` is in your path, or use fully qualified filepath to the `snapteld` binary: + ``` -$ $SNAP_PATH/bin/snapteld [global options] command [command options] [arguments...] +$ snapteld [global options] command [command options] [arguments...] ``` ### Options ``` ---disable-api, -d Disable the agent REST API ---api-port, -p '8181' API port (Default: 8181) ---log-level, -l '3' 1-5 (Debug, Info, Warning, Error, Fatal) [$SNAP_LOG_LEVEL] ---log-path, -o Path for logs. Empty path logs to stdout. [$SNAP_LOG_PATH] +--log-level value, -l value 1-5 (Debug, Info, Warning, Error, Fatal; default: 3) [$SNAP_LOG_LEVEL] +--log-path value, -o value Path for logs. Empty path logs to stdout. [$SNAP_LOG_PATH] --log-truncate Log file truncating mode. Default is false => append (true => truncate). --log-colors Log file coloring mode. Default is true => colored (--log-colors=false => no colors). ---max-procs, -c '1' Set max cores to use for Snap Agent. Default is 1 core. [$GOMAXPROCS] ---auto-discover, -a Auto discover paths separated by colons. [$SNAP_AUTODISCOVER_PATH] ---max-running-plugins, -m '3' The maximum number of instances of a loaded plugin to run [$SNAP_MAX_PLUGINS] ---plugin-load-timeout '3' The maximum number of seconds a plugin can take to load [$SNAP_PLUGIN_LOAD_TIMEOUT] ---cache-expiration '500ms' The time limit for which a metric cache entry is valid [$SNAP_CACHE_EXPIRATION] ---plugin-trust, -t '1' 0-2 (Disabled, Enabled, Warning) [$SNAP_TRUST_LEVEL] ---keyring-paths, -k Keyring paths for signing verification separated by colons [$SNAP_KEYRING_PATHS] ---rest-cert A path to a certificate to use for HTTPS deployment of Snap's REST API ---config A path to a config file +--max-procs value, -c value Set max cores to use for Snap Agent (default: 1) [$GOMAXPROCS] +--config value A path to a config file [$SNAP_CONFIG_PATH] +--max-running-plugins value, -m value The maximum number of instances of a loaded plugin to run (default: 3) [$SNAP_MAX_PLUGINS] +--plugin-load-timeout value The maximum number seconds a plugin can take to load (default: 3) [$SNAP_PLUGIN_LOAD_TIMEOUT] +--auto-discover value, -a value Auto discover paths separated by colons. [$SNAP_AUTODISCOVER_PATH] +--plugin-trust value, -t value 0-2 (Disabled, Enabled, Warning; default: 1) [$SNAP_TRUST_LEVEL] +--keyring-paths value, -k value Keyring paths for signing verification separated by colons [$SNAP_KEYRING_PATHS] +--cache-expiration value The time limit for which a metric cache entry is valid (default: 500ms) [$SNAP_CACHE_EXPIRATION] +--control-listen-port value Listen port for control RPC server (default: 8082) [$SNAP_CONTROL_LISTEN_PORT] +--control-listen-addr value Listen address for control RPC server [$SNAP_CONTROL_LISTEN_ADDR] +--work-manager-queue-size value Size of the work manager queue (default: 25) [$WORK_MANAGER_QUEUE_SIZE] +--work-manager-pool-size value Size of the work manager pool (default: 4) [$WORK_MANAGER_POOL_SIZE] +--disable-api, -d Disable the agent REST API +--api-addr value, -b value API Address[:port] to bind to/listen on. Default: empty string => listen on all interfaces [$SNAP_ADDR] +--api-port value, -p value API port (default: 8181) [$SNAP_PORT] --rest-https start Snap's API as https ---rest-key A path to a key file to use for HTTPS deployment of Snap's REST API +--rest-cert value A path to a certificate to use for HTTPS deployment of Snap's REST API +--rest-key value A path to a key file to use for HTTPS deployment of Snap's REST API --rest-auth Enables Snap's REST API authentication ---work-manager-queue-size "0" Size of the work manager queue (default: 25) [$WORK_MANAGER_QUEUE_SIZE] ---work-manager-pool-size "0" Size of the work manager pool (default 4) [$WORK_MANAGER_POOL_SIZE] ---tribe-node-name 'tjerniga-mac01.local' Name of this node in tribe cluster (default: hostname) [$SNAP_TRIBE_NODE_NAME] +--pprof Enables profiling tools +--tribe-node-name value Name of this node in tribe cluster (default: hostname) [$SNAP_TRIBE_NODE_NAME] --tribe Enable tribe mode [$SNAP_TRIBE] ---tribe-seed IP (or hostname) and port of a node to join (e.g. 127.0.0.1:6000) [$SNAP_TRIBE_SEED] ---tribe-addr '192.168.10.101' Addr tribe gossips over to maintain membership [$SNAP_TRIBE_ADDR] ---tribe-port '6000' Port tribe gossips over to maintain membership [$SNAP_TRIBE_PORT] +--tribe-seed value IP (or hostname) and port of a node to join (e.g. 127.0.0.1:6000) [$SNAP_TRIBE_SEED] +--tribe-addr value Addr tribe gossips over to maintain membership [$SNAP_TRIBE_ADDR] +--tribe-port value Port tribe gossips over to maintain membership (default: 6000) [$SNAP_TRIBE_PORT] --help, -h show help --version, -v print the version ``` ## Examples + ### Commands ``` -$SNAP_PATH/bin/snapteld -$SNAP_PATH/bin/snapteld -log-level 4 -$SNAP_PATH/bin/snapteld -l 1 -t 2 -k -$SNAP_PATH/bin/snapteld -a $SNAP_PATH/plugins/ -$SNAP_PATH/bin/snapteld --version +$ snapteld +$ snapteld --version +$ snapteld --log-level 4 +$ snapteld --auto-discover /opt/snap/plugins/ +$ snapteld --log-level 1 --plugin-trust 2 --keyring-paths /etc/snap/keyrings ``` -### Output -``` -$ $SNAP_PATH/bin/snapteld -l 1 -t 0 --rest-auth -``` +### Debug output +By default, Snap daemon loads the configuration in `/etc/snap/snapteld.conf` and writes logs to `/var/log/snap/snapteld.log`. When debugging Snap issues, instead of a daemon, you can run it as a foreground process to review the logs directly: + +* shutdown `snap-telemetry` service and ensure there's no Snap processes running: + RedHat 6/Ubuntu 14.04: + ``` +$ sevice snap-telemetry stop +$ pgrep snap + ``` + RedHat 7/Ubuntu 16.04: + ``` +$ systemctl stop snap-telemetry +$ pgrep snap + ``` + +* run Snap with debug log `--log-level 1`, no log file `--log-path ''`, along with any other custom startup options (it will use the snapteld.conf file settings if an option is omitted): + ``` +$ snapteld --log-level 1 --log-path '' --plugin-trust 0 + ``` + +This should result in the following log output: ``` -INFO[0000] Starting snapteld (version: unknown) -INFO[0000] setting GOMAXPROCS to: 1 core(s) -INFO[0000] control started _block=start _module=control -INFO[0000] module started _module=snapteld block=main snap-module=control -INFO[0000] scheduler started _block=start-scheduler _module=scheduler -INFO[0000] module started _module=snapteld block=main snap-module=scheduler -INFO[0000] setting plugin trust level to: disabled -INFO[0000] auto discover path is disabled -INFO[0000] Configuring REST API with HTTPS set to: false _module=_mgmt-rest -INFO[0000] REST API authentication is enabled -What password do you want to use for authentication? -Password: -INFO[0111] REST API authentication password is set -INFO[0111] Starting REST API on :8181 _module=_mgmt-rest -INFO[0111] REST API is enabled -INFO[0111] snapteld started _module=snapteld block=main -INFO[0111] setting log level to: debug +INFO[2017-01-09T12:55:05-08:00] setting log level to: debug +INFO[2017-01-09T12:55:05-08:00] Starting snapteld (version: 1.0.0) +INFO[2017-01-09T12:55:05-08:00] setting GOMAXPROCS to: 1 core(s) +... +DEBU[2017-01-09T12:55:05-08:00] metric manager linked _block=set-metric-manager _module=scheduler +INFO[2017-01-09T12:55:05-08:00] Configuring REST API with HTTPS set to: false _module=_mgmt-rest +INFO[2017-01-09T12:55:05-08:00] REST API is enabled +INFO[2017-01-09T12:55:05-08:00] control started _block=start _module=control +INFO[2017-01-09T12:55:05-08:00] auto discover path is disabled _block=start _module=control +INFO[2017-01-09T12:55:05-08:00] module started _module=snapteld block=main snap-module=control +INFO[2017-01-09T12:55:05-08:00] scheduler started _block=start-scheduler _module=scheduler +INFO[2017-01-09T12:55:05-08:00] auto discover path is disabled _block=start-scheduler _module=scheduler +INFO[2017-01-09T12:55:05-08:00] module started _module=snapteld block=main snap-module=scheduler +INFO[2017-01-09T12:55:05-08:00] Starting REST API on :8181 _module=_mgmt-rest +INFO[2017-01-09T12:55:05-08:00] REST started _block=start _module=_mgmt-rest +INFO[2017-01-09T12:55:05-08:00] module started _module=snapteld block=main snap-module=REST +INFO[2017-01-09T12:55:05-08:00] setting plugin trust level to: disabled +INFO[2017-01-09T12:55:05-08:00] snapteld started + ss ss + odyssyhhyo oyhysshhoyo + ddddyssyyysssssyyyyyyyssssyyysssyhy+- + ssssssooosyhhysssyyyyyyyyyyyyyyyyyyyyyyyyyyyyssyhh+. + ssss lhyssssssyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyssydo + sssssssssshhhhs lsyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyshh. + ssyyyysssssssssssyhdo syyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyd. + syyyyyyyyhhyyyyyyyyyyyyhdo syyyyyyyyyyyyydddhyyyyyyyyyyyyyhhhyyyyyyyyyyyyhh + ssyyyyyyyh hhyyyyyyyyyyyyhdo syyyyyyyyyyyddyddddhhhhhhhhdddhhddyyyyyyyyyyyydo + ddyyyyyyh | hyyyyyyyyyyyyydds syyyyyyyhhdhyyyhhhhhhhhhhyyyyyyhhdhyyyyyyyyyyym+ + dhyyyyyyyhhhhyyyyyyyyyyyyyyhmds shyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyydyyl + dhddhyyyyyyhdhyyyyyyyyyyyyyydhmo yhyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyhdhsh + dhyyhysshhdmdhyyyyyyyyyyyyyyhhdh hhyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyhhdyoohmh ylo + yy dmyyyyyyyyyyyyyyyyhhhm odhyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyhdhsoshdyyhdddy ylo + odhhyyyyyyyyyyyyyyyyyyhhdy oyhhhyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyhhmhyhdhhyyyyyhddoyddy ylo + dddhhyyyyyyyyyyyyyyyhhhyhhdhs ooosydhhhhyyyyyyyyyyyyyyyyyyyyhhhhhyso+oymyyyyyyyyyyyhhddydmmyys + ohdyyyyyyyyyyyyyyyhdhyyhhhddyoooohhhhhhhhhhhhhhhhhhhhhdhhyysooosyhhhhhyhhhhhhyyyhyyyhhhhddyy + dyhyyyyyyyyyyyyydhyyyyyhhdddoooooooooooooooooooooooyysyyhddddhhyyyyydmdddddddmmddddhyyy + dmmmmoNddddddmddhhhhyyyyyhhhdddddddhhhhhhhhhdddddddddhhhyyyyyyyyyyhNmmmooooooooyyy + Nhhhhhhhdmmddmhyyyyyyyyyhhhhhhhhhhhhhhhhhhhhyyyyyyyyyyyyyyyhhm + NhhhhhhhhhdmyhdyyyyyyyyyyyyhyyyyyyyyyyyyyyyhhhdhyyyyyyyyyyyhhN + NhyyyyyyyyyN dmyyyyyyyyyyhdmdhhhhhhhhhhdhhmmmmN NyyyyyyyyyyhhN + NhyyyyyyyyyN Nyyyyyyyyyhhm NmddmH Nyyyyyyyyyhdm + .d8888b. dmomomommmmh dhhhhhhyyhhmh NddddmH Nyyyyyyyyhdh + d88P Y88b dmomomommmmmh dmomoH dmomomommmmh + Y88b. + "Y888b. 88888b. 8888b. 88888b. + "Y88b. 888 "88b "88b 888 "88b + "888 888 888 .d888888 888 888 + Y88b d88P 888 888 888 888 888 d88P + "Y8888P" 888 888 "Y888888 88888P" + 888 + 888 + 888 _module=snapteld block=main ``` + ## More information * [SNAPTELD_CONFIGURATION.md](SNAPTELD_CONFIGURATION.md) * [REST_API.md](REST_API.md)