Merge branch 'agent_logging' into dev

README.md

@@ -133,7 +133,8 @@ additional details on each available environment variable.
 | `AWS_SECRET_ACCESS_KEY` | EXAMPLEKEY | The [secret key](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html) used by the agent for all calls. | Taken from Amazon EC2 instance metadata. | Taken from Amazon EC2 instance metadata. |
 | `AWS_SESSION_TOKEN` | | The [session token](http://docs.aws.amazon.com/STS/latest/UsingSTS/Welcome.html) used for temporary credentials. | Taken from Amazon EC2 instance metadata. | Taken from Amazon EC2 instance metadata. |
 | `DOCKER_HOST`   | `unix:///var/run/docker.sock` | Used to create a connection to the Docker daemon; behaves similarly to this environment variable as used by the Docker client. | `unix:///var/run/docker.sock` | `npipe:////./pipe/docker_engine` |
-| `ECS_LOGLEVEL`  | <crit> | <error> | <warn> | <info> | <debug> | The level of detail that should be logged. | info | info |
+| `ECS_LOGLEVEL`  | <crit> | <error> | <warn> | <info> | <debug> | The level of detail that should be logged by the logging driver, or on Windows, the Windows Event Log. | info | info |
+| `ECS_LOGLEVEL_ON_INSTANCE`  | <none> | <crit> | <error> | <warn> | <info> | <debug> | The level of detail that should be logged in the on-instance log file. | none if `ECS_LOG_DRIVER` is explicitly set to a non-empty value; info otherwise | none if `ECS_LOG_DRIVER` is explicitly set to a non-empty value; info otherwise |
 | `ECS_LOGFILE`   | /ecs-agent.log              | The location where logs should be written. Log level is controlled by `ECS_LOGLEVEL`. | blank | blank |
 | `ECS_CHECKPOINT`   | <true | false> | Whether to checkpoint state to the DATADIR specified below. | true if `ECS_DATADIR` is explicitly set to a non-empty value; false otherwise | true if `ECS_DATADIR` is explicitly set to a non-empty value; false otherwise |
 | `ECS_DATADIR`      |   /data/                  | The container path where state is checkpointed for use across agent restarts. Note that on Linux, when you specify this, you will need to make sure that the Agent container has a bind mount of `$ECS_HOST_DATA_DIR/data:$ECS_DATADIR` with the corresponding values of `ECS_HOST_DATA_DIR` and `ECS_DATADIR`. | /data/ | `C:\ProgramData\Amazon\ECS\data`
@@ -184,6 +185,8 @@ additional details on each available environment variable.
 | `ECS_LOG_OUTPUT_FORMAT` | `logfmt` | `json` | Determines the log output format. When the json format is used, each line in the log would be a structured JSON map. | `logfmt` | `logfmt` |
 | `ECS_LOG_MAX_FILE_SIZE_MB` | `10` | When the ECS_LOG_ROLLOVER_TYPE variable is set to size, this variable determines the maximum size (in MB) the log file before it is rotated. If the rollover type is set to hourly then this variable is ignored. | `10` | `10` |
 | `ECS_LOG_MAX_ROLL_COUNT` | `24` | Determines the number of rotated log files to keep. Older log files are deleted once this limit is reached. | `24` | `24` |
+| `ECS_LOG_DRIVER` | `awslogs` | `fluentd` | `gelf` | `json-file` | `journald` | `logentries` | `syslog` | `splunk` | The logging driver to be used by the Agent container. | `json-file` | Not applicable |
+| `ECS_LOG_OPTS` | `{"option":"value"}` | The options for configuring the logging driver set in `ECS_LOG_DRIVER`. | `{}` | Not applicable |
 | `ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE` | `true` | Whether to enable awslogs log driver to authenticate via credentials of task execution IAM role. Needs to be true if you want to use awslogs log driver in a task that has task execution IAM role specified. When using the ecs-init RPM with version equal or later than V1.16.0-1, this env is set to true by default. | `false` | `false` |
 
 ### Persistence

agent/app/args/flag.go

@@ -13,11 +13,15 @@
 
 package args
 
-import "flag"
+import (
+	"flag"
+)
 
 const (
 	versionUsage             = "Print the agent version information and exit"
-	logLevelUsage            = "Loglevel: [<crit>|<error>|<warn>|<info>|<debug>]"
+	logLevelUsage            = "Loglevel overrides loglevel-driver and loglevel-on-instance and sets the same level for both on-instance and driver logging: [<crit>|<error>|<warn>|<info>|<debug>]"
+	driverLogLevelUsage      = "Loglevel for docker logging driver: [<crit>|<error>|<warn>|<info>|<debug>]"
+	instanceLogLevelUsage    = "Loglevel for Agent on-instance log file: [<none>|<crit>|<error>|<warn>|<info>|<debug>]"
 	ecsAttributesUsage       = "Print the Agent's ECS Attributes based on its environment"
 	acceptInsecureCertUsage  = "Disable SSL certificate verification. We do not recommend setting this option"
 	licenseUsage             = "Print the LICENSE and NOTICE files and exit"
@@ -27,6 +31,8 @@ const (
 
 	versionFlagName              = "version"
 	logLevelFlagName             = "loglevel"
+	driverLogLevelFlagName       = "loglevel-driver"
+	instanceLogLevelFlagName     = "loglevel-on-instance"
 	ecsAttributesFlagName        = "ecs-attributes"
 	acceptInsecureCertFlagName   = "k"
 	licenseFlagName              = "license"
@@ -39,8 +45,12 @@ const (
 type Args struct {
 	// Version indicates if the version information should be printed
 	Version *bool
-	// LogLevel represents the ECS Agent's log level
+	// LogLevel represents the ECS Agent's logging level for both the on-instance file and the logging driver
 	LogLevel *string
+	// DriverLogLevel represents the ECS Agent's logging driver log level
+	DriverLogLevel *string
+	// InstanceLogLevel represents the ECS Agent's on-instance file log level
+	InstanceLogLevel *string
 	// AcceptInsecureCert indicates if SSL certificate verification
 	// should be disabled
 	AcceptInsecureCert *bool
@@ -64,6 +74,8 @@ func New(arguments []string) (*Args, error) {
 	args := &Args{
 		Version:              flagset.Bool(versionFlagName, false, versionUsage),
 		LogLevel:             flagset.String(logLevelFlagName, "", logLevelUsage),
+		DriverLogLevel:       flagset.String(driverLogLevelFlagName, "", driverLogLevelUsage),
+		InstanceLogLevel:     flagset.String(instanceLogLevelFlagName, "", instanceLogLevelUsage),
 		AcceptInsecureCert:   flagset.Bool(acceptInsecureCertFlagName, false, acceptInsecureCertUsage),
 		License:              flagset.Bool(licenseFlagName, false, licenseUsage),
 		BlackholeEC2Metadata: flagset.Bool(blackholeEC2MetadataFlagName, false, blacholeEC2MetadataUsage),

agent/app/run.go

@@ -46,7 +46,11 @@ func Run(arguments []string) int {
 		return runHealthcheck("http://localhost:51678/v1/metadata", time.Second*25)
 	}
 
-	logger.SetLevel(*parsedArgs.LogLevel)
+	if *parsedArgs.LogLevel != "" {
+		logger.SetLevel(*parsedArgs.LogLevel, *parsedArgs.LogLevel)
+	} else {
+		logger.SetLevel(*parsedArgs.DriverLogLevel, *parsedArgs.InstanceLogLevel)
+	}
 
 	// Create an Agent object
 	agent, err := newAgent(aws.BoolValue(parsedArgs.BlackholeEC2Metadata), parsedArgs.AcceptInsecureCert)

agent/logger/eventlog_windows.go

@@ -64,7 +64,7 @@ func registerPlatformLogger() {
 // platformLogConfig exposes log configuration for the event log receiver
 func platformLogConfig() string {
 	return `
-		<custom name="wineventlog" formatid="windows" />`
+			<custom name="wineventlog" formatid="windows" />`
 }
 
 // ReceiveMessage receives a log line from seelog and emits it to the Windows event log

agent/logger/log.go

@@ -25,26 +25,30 @@ import (
 )
 
 const (
-	LOGLEVEL_ENV_VAR           = "ECS_LOGLEVEL"
-	LOGFILE_ENV_VAR            = "ECS_LOGFILE"
-	LOG_ROLLOVER_TYPE_ENV_VAR  = "ECS_LOG_ROLLOVER_TYPE"
-	LOG_OUTPUT_FORMAT_ENV_VAR  = "ECS_LOG_OUTPUT_FORMAT"
-	LOG_MAX_FILE_SIZE_ENV_VAR  = "ECS_LOG_MAX_FILE_SIZE_MB"
-	LOG_MAX_ROLL_COUNT_ENV_VAR = "ECS_LOG_MAX_ROLL_COUNT"
-
-	DEFAULT_LOGLEVEL               = "info"
-	DEFAULT_ROLLOVER_TYPE          = "date"
-	DEFAULT_OUTPUT_FORMAT          = "logfmt"
-	DEFAULT_MAX_FILE_SIZE  float64 = 10
-	DEFAULT_MAX_ROLL_COUNT int     = 24
+	LOGLEVEL_ENV_VAR             = "ECS_LOGLEVEL"
+	LOGLEVEL_ON_INSTANCE_ENV_VAR = "ECS_LOGLEVEL_ON_INSTANCE"
+	LOGFILE_ENV_VAR              = "ECS_LOGFILE"
+	LOG_DRIVER_ENV_VAR           = "ECS_LOG_DRIVER"
+	LOG_ROLLOVER_TYPE_ENV_VAR    = "ECS_LOG_ROLLOVER_TYPE"
+	LOG_OUTPUT_FORMAT_ENV_VAR    = "ECS_LOG_OUTPUT_FORMAT"
+	LOG_MAX_FILE_SIZE_ENV_VAR    = "ECS_LOG_MAX_FILE_SIZE_MB"
+	LOG_MAX_ROLL_COUNT_ENV_VAR   = "ECS_LOG_MAX_ROLL_COUNT"
+
+	DEFAULT_LOGLEVEL                         = "info"
+	DEFAULT_LOGLEVEL_WHEN_DRIVER_SET         = "off"
+	DEFAULT_ROLLOVER_TYPE                    = "date"
+	DEFAULT_OUTPUT_FORMAT                    = "logfmt"
+	DEFAULT_MAX_FILE_SIZE            float64 = 10
+	DEFAULT_MAX_ROLL_COUNT           int     = 24
 )
 
 type logConfig struct {
 	RolloverType  string
 	MaxRollCount  int
 	MaxFileSizeMB float64
 	logfile       string
-	level         string
+	driverLevel   string
+	instanceLevel string
 	outputFormat  string
 	lock          sync.Mutex
 }
@@ -76,34 +80,52 @@ func reloadConfig() {
 
 func seelogConfig() string {
 	c := `
-<seelog type="asyncloop" minlevel="` + Config.level + `">
+<seelog type="asyncloop">
 	<outputs formatid="` + Config.outputFormat + `">
-		<console />`
+		<filter levels="` + getLevelList(Config.driverLevel) + `">
+			<console />`
 	c += platformLogConfig()
+	c += `
+		</filter>
+		<filter levels="` + getLevelList(Config.instanceLevel) + `">`
 	if Config.logfile != "" {
 		if Config.RolloverType == "size" {
 			c += `
-		<rollingfile filename="` + Config.logfile + `" type="size"
-		 maxsize="` + strconv.Itoa(int(Config.MaxFileSizeMB*1000000)) + `" archivetype="none" maxrolls="` + strconv.Itoa(Config.MaxRollCount) + `" />`
+			<rollingfile filename="` + Config.logfile + `" type="size"
+			 maxsize="` + strconv.Itoa(int(Config.MaxFileSizeMB*1000000)) + `" archivetype="none" maxrolls="` + strconv.Itoa(Config.MaxRollCount) + `" />`
 		} else {
 			c += `
-		<rollingfile filename="` + Config.logfile + `" type="date"
-		 datepattern="2006-01-02-15" archivetype="none" maxrolls="` + strconv.Itoa(Config.MaxRollCount) + `" />`
+			<rollingfile filename="` + Config.logfile + `" type="date"
+			 datepattern="2006-01-02-15" archivetype="none" maxrolls="` + strconv.Itoa(Config.MaxRollCount) + `" />`
 		}
 	}
 	c += `
+		</filter>
 	</outputs>
 	<formats>
 		<format id="logfmt" format="%EcsAgentLogfmt" />
 		<format id="json" format="%EcsAgentJson" />
 		<format id="windows" format="%Msg" />
 	</formats>
 </seelog>`
+
 	return c
 }
 
-// SetLevel sets the log level for logging
-func SetLevel(logLevel string) {
+func getLevelList(fileLevel string) string {
+	levelLists := map[string]string{
+		"debug":    "debug,info,warn,error,critical",
+		"info":     "info,warn,error,critical",
+		"warn":     "warn,error,critical",
+		"error":    "error,critical",
+		"critical": "critical",
+		"off":      "off",
+	}
+	return levelLists[fileLevel]
+}
+
+// SetLevel sets the log levels for logging
+func SetLevel(driverLogLevel, instanceLogLevel string) {
 	levels := map[string]string{
 		"debug": "debug",
 		"info":  "info",
@@ -112,12 +134,19 @@ func SetLevel(logLevel string) {
 		"crit":  "critical",
 		"none":  "off",
 	}
-	parsedLevel, ok := levels[strings.ToLower(logLevel)]
 
-	if ok {
+	parsedDriverLevel, driverOk := levels[strings.ToLower(driverLogLevel)]
+	parsedInstanceLevel, instanceOk := levels[strings.ToLower(instanceLogLevel)]
+
+	if instanceOk || driverOk {
 		Config.lock.Lock()
 		defer Config.lock.Unlock()
-		Config.level = parsedLevel
+		if instanceOk {
+			Config.instanceLevel = parsedInstanceLevel
+		}
+		if driverOk {
+			Config.driverLevel = parsedDriverLevel
+		}
 		reloadConfig()
 	}
 }
@@ -127,13 +156,21 @@ func GetLevel() string {
 	Config.lock.Lock()
 	defer Config.lock.Unlock()
 
-	return Config.level
+	return Config.driverLevel
+}
+
+func setInstanceLevelDefault() string {
+	if logDriver := os.Getenv(LOG_DRIVER_ENV_VAR); logDriver != "" {
+		return DEFAULT_LOGLEVEL_WHEN_DRIVER_SET
+	}
+	return DEFAULT_LOGLEVEL
 }
 
 func init() {
 	Config = &logConfig{
 		logfile:       os.Getenv(LOGFILE_ENV_VAR),
-		level:         DEFAULT_LOGLEVEL,
+		driverLevel:   DEFAULT_LOGLEVEL,
+		instanceLevel: setInstanceLevelDefault(),
 		RolloverType:  DEFAULT_ROLLOVER_TYPE,
 		outputFormat:  DEFAULT_OUTPUT_FORMAT,
 		MaxFileSizeMB: DEFAULT_MAX_FILE_SIZE,
@@ -147,7 +184,8 @@ func init() {
 		seelog.Error(err)
 	}
 
-	SetLevel(os.Getenv(LOGLEVEL_ENV_VAR))
+	SetLevel(os.Getenv(LOGLEVEL_ENV_VAR), os.Getenv(LOGLEVEL_ON_INSTANCE_ENV_VAR))
+
 	if RolloverType := os.Getenv(LOG_ROLLOVER_TYPE_ENV_VAR); RolloverType != "" {
 		Config.RolloverType = RolloverType
 	}