diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index e455c0102b3..376ceb4190f 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -2,7 +2,7 @@ @Library('apm@current') _ pipeline { - agent { label 'linux && immutable' } + agent none environment { REPO = 'apm-server' BASE_DIR = "src/github.com/elastic/${env.REPO}" @@ -27,63 +27,67 @@ pipeline { } triggers { issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*') + upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }") } stages { - /** - Checkout the code and stash it, to use it on other stages. - */ - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - deleteDir() - gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false) - script { - dir("${BEATS_DIR}"){ - env.GO_VERSION = readFile(".go-version").trim() - def regexps =[ - "^devtools/mage.*", - "^libbeat/scripts/Makefile", - ] - env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps) - // Skip all the stages except docs for PR's with asciidoc changes only - env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true) - } - } - } - } - /** - updates beats updates the framework part and go parts of beats. - Then build and test. - Finally archive the results. - */ - stage('Update Beats') { - options { skipDefaultCheckout() } + stage('Filter build') { + agent { label 'ubuntu && immutable' } when { beforeAgent true - anyOf { - branch 'master' - branch "\\d+\\.\\d+" - branch "v\\d?" - tag "v\\d+\\.\\d+\\.\\d+*" - allOf { - expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() } - changeRequest() + expression { + return isCommentTrigger() || isUserTrigger() + } + } + /** + Checkout the code and stash it, to use it on other stages. + */ + stage('Checkout') { + steps { + deleteDir() + gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false) + script { + dir("${BEATS_DIR}"){ + env.GO_VERSION = readFile(".go-version").trim() + def regexps =[ + "^devtools/mage.*", + "^libbeat/scripts/Makefile", + ] + env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps) + // Skip all the stages except docs for PR's with asciidoc changes only + env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true) + } } - } } - steps { - withGithubNotify(context: 'Check Apm Server Beats Update') { - beatsUpdate() + /** + updates beats updates the framework part and go parts of beats. + Then build and test. + Finally archive the results. + */ + stage('Update Beats') { + options { skipDefaultCheckout() } + when { + beforeAgent true + anyOf { + branch 'master' + branch "\\d+\\.\\d+" + branch "v\\d?" + tag "v\\d+\\.\\d+\\.\\d+*" + allOf { + expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() } + changeRequest() + } + + } + } + steps { + withGithubNotify(context: 'Check Apm Server Beats Update') { + beatsUpdate() + } } } } } - // post { - // cleanup { - // notifyBuildResult() - // } - // } } def beatsUpdate() { diff --git a/.ci/build-docker-images.groovy b/.ci/build-docker-images.groovy index f093acba52c..417e2fefae4 100644 --- a/.ci/build-docker-images.groovy +++ b/.ci/build-docker-images.groovy @@ -3,7 +3,7 @@ @Library('apm@current') _ pipeline { - agent { label 'linux && immutable' } + agent { label 'ubuntu-16 && immutable' } environment { REPO = 'beats' BASE_DIR = "src/github.com/elastic/${env.REPO}" diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml index f612aa5a665..fd38576b387 100644 --- a/.ci/jobs/apm-beats-update.yml +++ b/.ci/jobs/apm-beats-update.yml @@ -5,7 +5,6 @@ view: Beats concurrent: true project-type: multibranch - periodic-folder-trigger: 1w prune-dead-branches: true number-to-keep: 10 days-to-keep: 30 @@ -19,20 +18,19 @@ discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' discover-tags: true + head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)' disable-pr-notifications: true notification-context: 'apm-beats-update' - property-strategies: - all-branches: - - suppress-scm-triggering: true repo: 'beats' repo-owner: 'elastic' credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken ssh-checkout: credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba build-strategies: + - skip-initial-build: true - tags: ignore-tags-older-than: -1 - ignore-tags-newer-than: -1 + ignore-tags-newer-than: 30 - named-branches: - exact-name: name: 'master' diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml index 076a8a81bbe..5e0bee6b03f 100644 --- a/.ci/jobs/beats.yml +++ b/.ci/jobs/beats.yml @@ -6,7 +6,6 @@ view: Beats concurrent: true project-type: multibranch - periodic-folder-trigger: 1w prune-dead-branches: true number-to-keep: 10 days-to-keep: 30 @@ -19,6 +18,7 @@ discover-pr-forks-strategy: 'merge-current' discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' + head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)' discover-tags: true notification-context: "beats-ci" property-strategies: diff --git a/.ci/jobs/build-it-docker-images.yml b/.ci/jobs/build-it-docker-images.yml index 562af8d1101..79a14779e6e 100644 --- a/.ci/jobs/build-it-docker-images.yml +++ b/.ci/jobs/build-it-docker-images.yml @@ -16,7 +16,7 @@ - git: url: git@github.com:elastic/beats.git refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' + wipe-workspace: true name: origin shallow-clone: true credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml index f93e0cf52db..40d416f28fc 100644 --- a/.ci/jobs/defaults.yml +++ b/.ci/jobs/defaults.yml @@ -1,4 +1,4 @@ - + --- ##### GLOBAL METADATA @@ -16,5 +16,4 @@ publishers: - email: recipients: infra-root+build@elastic.co - periodic-folder-trigger: 1w prune-dead-branches: true diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml index 848ca742aed..5aa8967e2f7 100644 --- a/.ci/jobs/packaging.yml +++ b/.ci/jobs/packaging.yml @@ -14,7 +14,8 @@ discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' discover-tags: true - disable-pr-notifications: false + head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)' + disable-pr-notifications: true notification-context: 'beats-packaging' repo: 'beats' repo-owner: 'elastic' @@ -22,9 +23,10 @@ ssh-checkout: credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba build-strategies: + - skip-initial-build: true - tags: - ignore-tags-older-than: 30 - ignore-tags-newer-than: -1 + ignore-tags-older-than: -1 + ignore-tags-newer-than: 30 - named-branches: - exact-name: name: 'master' diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 25ec166fd83..81d67809272 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -3,7 +3,7 @@ @Library('apm@current') _ pipeline { - agent { label 'ubuntu && immutable' } + agent none environment { BASE_DIR = 'src/github.com/elastic/beats' JOB_GCS_BUCKET = 'beats-ci-artifacts' @@ -12,6 +12,7 @@ pipeline { DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_REGISTRY = 'docker.elastic.co' SNAPSHOT = "true" + PIPELINE_LOG_LEVEL = "INFO" } options { timeout(time: 3, unit: 'HOURS') @@ -32,106 +33,105 @@ pipeline { booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.') } stages { - stage('Checkout') { + stage('Filter build') { + agent { label 'ubuntu && immutable' } when { beforeAgent true - not { - triggeredBy 'SCMTrigger' + expression { + return isCommentTrigger() || isUserTrigger() || isUpstreamTrigger() } } - options { skipDefaultCheckout() } - steps { - deleteDir() - gitCheckout(basedir: "${BASE_DIR}") - setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") - } - } - stage('Build Packages'){ - when { - beforeAgent true - not { - triggeredBy 'SCMTrigger' - } - } - matrix { - axes { - axis { - name 'BEATS_FOLDER' - values ( - 'auditbeat', - 'filebeat', - 'heartbeat', - 'journalbeat', - 'metricbeat', - 'packetbeat', - 'winlogbeat', - 'x-pack/auditbeat', - 'x-pack/elastic-agent', - 'x-pack/dockerlogbeat', - 'x-pack/filebeat', - 'x-pack/functionbeat', - // 'x-pack/heartbeat', - // 'x-pack/journalbeat', - 'x-pack/metricbeat', - // 'x-pack/packetbeat', - 'x-pack/winlogbeat' - ) + stages { + stage('Checkout') { + options { skipDefaultCheckout() } + steps { + deleteDir() + gitCheckout(basedir: "${BASE_DIR}") + setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) + stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") } } - stages { - stage('Package Linux'){ - agent { label 'ubuntu && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.linux - } - } - environment { - HOME = "${env.WORKSPACE}" - PLATFORMS = [ - '+all', - 'linux/amd64', - 'linux/386', - 'linux/arm64', - 'linux/armv7', - 'linux/ppc64le', - 'linux/mips64', - 'linux/s390x', - 'windows/amd64', - 'windows/386', - (params.macos ? '' : 'darwin/amd64'), - ].join(' ') - } - steps { - withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") { - release() - pushCIDockerImages() + stage('Build Packages'){ + matrix { + axes { + axis { + name 'BEATS_FOLDER' + values ( + 'auditbeat', + 'filebeat', + 'heartbeat', + 'journalbeat', + 'metricbeat', + 'packetbeat', + 'winlogbeat', + 'x-pack/auditbeat', + 'x-pack/elastic-agent', + 'x-pack/dockerlogbeat', + 'x-pack/filebeat', + 'x-pack/functionbeat', + // 'x-pack/heartbeat', + // 'x-pack/journalbeat', + 'x-pack/metricbeat', + // 'x-pack/packetbeat', + 'x-pack/winlogbeat' + ) } } - } - stage('Package Mac OS'){ - agent { label 'macosx-10.12' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.macos + stages { + stage('Package Linux'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.linux + } + } + environment { + HOME = "${env.WORKSPACE}" + PLATFORMS = [ + '+all', + 'linux/amd64', + 'linux/386', + 'linux/arm64', + 'linux/armv7', + 'linux/ppc64le', + 'linux/mips64', + 'linux/s390x', + 'windows/amd64', + 'windows/386', + (params.macos ? '' : 'darwin/amd64'), + ].join(' ') + } + steps { + withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") { + release() + pushCIDockerImages() + } + } } - } - environment { - HOME = "${env.WORKSPACE}" - PLATFORMS = [ - '+all', - 'darwin/amd64', - ].join(' ') - } - steps { - withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") { - withMacOSEnv(){ - release() + stage('Package Mac OS'){ + agent { label 'macosx-10.12' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macos + } + } + environment { + HOME = "${env.WORKSPACE}" + PLATFORMS = [ + '+all', + 'darwin/amd64', + ].join(' ') + } + steps { + withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") { + withMacOSEnv(){ + release() + } + } } } } diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d2274bb0080..d1ab5687dc9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -156,6 +156,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit - Add Kibana Dashboard for MISP module. {pull}14147[14147] - Add support for gzipped files in S3 input {pull}13980[13980] - Add Filebeat Azure Dashboards {pull}14127[14127] +- Add support for thread ID in Filebeat Kafka module. {pull}19463[19463] *Heartbeat* diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6bcd7474392..a49e352f600 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -53,6 +53,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Okta module now requires objects instead of JSON strings for the `http_headers`, `http_request_body`, `pagination`, `rate_limit`, and `ssl` variables. {pull}18953[18953] - Adds oauth support for httpjson input. {issue}18415[18415] {pull}18892[18892] - Adds `split_events_by` option to httpjson input. {pull}19246[19246] +- Adds `date_cursor` option to httpjson input. {pull}19483[19483] *Heartbeat* @@ -197,6 +198,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Cisco ASA dissect pattern for 313008 & 313009 messages. {pull}19149[19149] - Fix date and timestamp formats for fortigate module {pull}19316[19316] - Fix memory leak in tcp and unix input sources. {pull}19459[19459] +- Add missing `default_field: false` to aws filesets fields.yml. {pull}19568[19568] +- Fix tls mapping in suricata module {issue}19492[19492] {pull}19494[19494] *Heartbeat* @@ -258,7 +261,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix incorrect usage of hints builder when exposed port is a substring of the hint {pull}19052[19052] - Remove dedot for tag values in aws module. {issue}19112[19112] {pull}19221[19221] - Stop counterCache only when already started {pull}19103[19103] -- Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] +- Set tags correctly if the dimension value is ARN {issue}19111[19111] {pull}19433[19433] - Fix bug incorrect parsing of float numbers as integers in Couchbase module {issue}18949[18949] {pull}19055[19055] *Packetbeat* @@ -310,8 +313,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958] - Add backoff configuration options for the Kafka output. {issue}16777[16777] {pull}17808[17808] - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607] +- Change ownership of files in docker images so they can be used in secured environments. {pull}12905[12905] - Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817] - Add support for multiple sets of hints on autodiscover {pull}18883[18883] +- Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181] *Auditbeat* @@ -417,6 +422,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Explicitly set ECS version in all Filebeat modules. {pull}19198[19198] - Add new mode to multiline reader to aggregate constant number of lines {pull}18352[18352] - Add automatic retries and exponential backoff to httpjson input. {pull}18956[18956] +- Add awscloudwatch input. {pull}19025[19025] - Changed the panw module to pass through (rather than drop) message types other than threat and traffic. {issue}16815[16815] {pull}19375[19375] - Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346] - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379] @@ -513,9 +519,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add memory metrics into compute googlecloud. {pull}18802[18802] - Add new fields to HAProxy module. {issue}18523[18523] - Add Tomcat overview dashboard {pull}14026[14026] +- Accept prefix as metric_types config parameter in googlecloud stackdriver metricset. {pull}19345[19345] - Update Couchbase to version 6.5 {issue}18595[18595] {pull}19055[19055] - Add dashboards for googlecloud load balancing metricset. {pull}18369[18369] - Add support for v1 consumer API in Cloud Foundry module, use it by default. {pull}19268[19268] +- Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] *Packetbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 02231608a01..e0f12058de4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -193,7 +193,7 @@ pipeline { when { beforeAgent true expression { - return env.BUILD_XPACK_FILEBEAT != "false" && params.macosTest + return env.BUILD_FILEBEAT_XPACK != "false" && params.macosTest } } steps { @@ -1104,7 +1104,7 @@ def isChanged(patterns){ def isChangedOSSCode(patterns) { def allPatterns = [ "^Jenkinsfile", - "^vendor/.*", + "^go.mod", "^libbeat/.*", "^testing/.*", "^dev-tools/.*", @@ -1117,7 +1117,7 @@ def isChangedOSSCode(patterns) { def isChangedXPackCode(patterns) { def allPatterns = [ "^Jenkinsfile", - "^vendor/.*", + "^go.mod", "^libbeat/.*", "^dev-tools/.*", "^testing/.*", @@ -1236,50 +1236,50 @@ def loadConfigEnvVars(){ // Auditbeat depends on metricbeat as framework, but does not include any of // the modules from Metricbeat. // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. - env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat')) - env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat')) + env.BUILD_AUDITBEAT = isChangedOSSCode(getProjectDependencies('auditbeat')) + env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/auditbeat')) // Dockerlogbeat is a standalone Beat that only relies on libbeat. - env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat')) + env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/dockerlogbeat')) // Filebeat depends on libbeat only. // The Filebeat x-pack build contains all functionality from OSS Filebeat. - env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat')) - env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat')) + env.BUILD_FILEBEAT = isChangedOSSCode(getProjectDependencies('filebeat')) + env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/filebeat')) // Metricbeat depends on libbeat only. // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. - env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat')) - env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat')) + env.BUILD_METRICBEAT = isChangedOSSCode(getProjectDependencies('metricbeat')) + env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/metricbeat')) // Functionbeat is a standalone beat that depends on libbeat only. // Functionbeat is available as x-pack build only. - env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat')) + env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/functionbeat')) // Heartbeat depends on libbeat only. // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. - env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat')) - env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat')) + env.BUILD_HEARTBEAT = isChangedOSSCode(getProjectDependencies('heartbeat')) + env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/heartbeat')) // Journalbeat depends on libbeat only. // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. - env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat')) - env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat')) + env.BUILD_JOURNALBEAT = isChangedOSSCode(getProjectDependencies('journalbeat')) + env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/journalbeat')) // Packetbeat depends on libbeat only. // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. - env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat')) - env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat')) + env.BUILD_PACKETBEAT = isChangedOSSCode(getProjectDependencies('packetbeat')) + env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/packetbeat')) // Winlogbeat depends on libbeat only. // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. - env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat')) - env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat')) + env.BUILD_WINLOGBEAT = isChangedOSSCode(getProjectDependencies('winlogbeat')) + env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/winlogbeat')) // Elastic-agent is a self-contained product, that depends on libbeat only. // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. // The agent is available as x-pack build only. - env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/elastic-agent')) + env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/elastic-agent')) // The Kubernetes test use Filebeat and Metricbeat, but only need to be run // if the deployment scripts have been updated. No Beats specific testing is @@ -1287,8 +1287,8 @@ def loadConfigEnvVars(){ env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) def generatorPatterns = ['^generator/.*'] - generatorPatterns.addAll(getVendorPatterns('generator/common/beatgen')) - generatorPatterns.addAll(getVendorPatterns('metricbeat/beater')) + generatorPatterns.addAll(getProjectDependencies('generator/common/beatgen')) + generatorPatterns.addAll(getProjectDependencies('metricbeat/beater')) env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) // Skip all the stages for changes only related to the documentation @@ -1327,7 +1327,7 @@ def isDocChangedOnly(){ /** This method grab the dependencies of a Go module and transform them on regexp */ -def getVendorPatterns(beatName){ +def getProjectDependencies(beatName){ def os = goos() def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" def output = "" @@ -1337,9 +1337,10 @@ def getVendorPatterns(beatName){ "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", ]) { output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - go list -mod=mod -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\ - |awk '{print \$1"/.*"}'\ - |sed -e "s#github.com/elastic/beats/v7/##g" + go list -deps ./${beatName} \ + | grep 'elastic/beats' \ + | sed -e "s#github.com/elastic/beats/v7/##g" \ + | awk '{print "^" \$1 "/.*"}' """) } return output?.split('\n').collect{ item -> item as String } diff --git a/Makefile b/Makefile index 10f7fd476cb..0e21f801f10 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PYTHON_ENV?=$(BUILD_DIR)/python-env PYTHON_EXE?=python3 PYTHON_ENV_EXE=${PYTHON_ENV}/bin/$(notdir ${PYTHON_EXE}) VENV_PARAMS?= -FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*" +FIND=find . -type f -not -path "*/build/*" -not -path "*/.git/*" GOLINT=golint GOLINT_REPO=golang.org/x/lint/golint REVIEWDOG=reviewdog @@ -96,8 +96,8 @@ clean: mage .PHONY: check check: python-env @$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;) - @$(FIND) -name *.py -name *.py -not -path "*/build/*" -not -path "*/vendor/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) - @$(FIND) -name *.py -not -path "*/build/*" -not -path "*/vendor/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) + @$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) + @$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) @# Validate that all updates were committed @$(MAKE) update @$(MAKE) check-headers diff --git a/NOTICE.txt b/NOTICE.txt index 7cf8cd112fe..97ddd49332d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2034,11 +2034,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/sarama -Version: v1.24.1-elastic.0.20200519143807-cbc80333a91e +Version: v1.19.1-0.20200629123429-0e7b69039eec Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.24.1-elastic.0.20200519143807-cbc80333a91e/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20200629123429-0e7b69039eec/LICENSE: Copyright (c) 2013 Shopify @@ -5936,11 +5936,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-concert -Version: v0.0.2 +Version: v0.0.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.0.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.0.3/LICENSE: Apache License Version 2.0, January 2004 @@ -13548,12 +13548,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/urso/ecslog -Version: v0.0.1 +Dependency : github.com/urso/sderr +Version: v0.0.0-20200210124243-c2a16f3d43ec Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/ecslog@v0.0.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: Apache License Version 2.0, January 2004 @@ -19995,37 +19995,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/alcortesm/tgz -Version: v0.0.0-20161220082320-9c5fe88206d7 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/alcortesm/tgz@v0.0.0-20161220082320-9c5fe88206d7/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2016 Alberto Cortés - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/alecthomas/template Version: v0.0.0-20190718012654-fb15b899a751 @@ -20092,36 +20061,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/anmitsu/go-shlex -Version: v0.0.0-20161002113705-648efa622239 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/anmitsu/go-shlex@v0.0.0-20161002113705-648efa622239/LICENSE: - -Copyright (c) anmitsu - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/antihax/optional Version: v0.0.0-20180407024304-ca021399b1a6 @@ -22917,39 +22856,6 @@ third-party archives. limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/creack/pty -Version: v1.1.7 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/creack/pty@v1.1.7/LICENSE: - -Copyright (c) 2011 Keith Rarick - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, -sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall -be included in all copies or substantial portions of the -Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/cucumber/godog Version: v0.8.1 @@ -24345,275 +24251,224 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/emirpasic/gods -Version: v1.12.0 -Licence type (autodetected): BSD-2-Clause +Dependency : github.com/envoyproxy/go-control-plane +Version: v0.9.4 +Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/emirpasic/gods@v1.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/go-control-plane@v0.9.4/LICENSE: -Copyright (c) 2015, Emir Pasic -All rights reserved. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. + 1. Definitions. -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -------------------------------------------------------------------------------- + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -AVL Tree: + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. -Copyright (c) 2017 Benjamin Scher Purcell + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -------------------------------------------------------------------------------- -Dependency : github.com/envoyproxy/go-control-plane -Version: v0.9.4 +Dependency : github.com/envoyproxy/protoc-gen-validate +Version: v0.1.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/go-control-plane@v0.9.4/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/envoyproxy/protoc-gen-validate -Version: v0.1.0 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-validate@v0.1.0/LICENSE: - +Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-validate@v0.1.0/LICENSE: + Apache License Version 2.0, January 2004 @@ -24853,218 +24708,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/flynn/go-shlex -Version: v0.0.0-20150515145356-3f9db97f8568 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/flynn/go-shlex@v0.0.0-20150515145356-3f9db97f8568/COPYING: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/fortytw2/leaktest Version: v1.3.0 @@ -25193,43 +24836,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/gliderlabs/ssh -Version: v0.2.2 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/gliderlabs/ssh@v0.2.2/LICENSE: - -Copyright (c) 2016 Glider Labs. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Glider Labs nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/go-gl/glfw/v3.3/glfw Version: v0.0.0-20191125211704-12ad95a8df72 @@ -31758,37 +31364,6 @@ See the License for the specific language governing permissions and limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/jbenet/go-context -Version: v0.0.0-20150711004518-d14ea06fba99 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/jbenet/go-context@v0.0.0-20150711004518-d14ea06fba99/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Juan Batiz-Benet - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/jcmturner/gofork Version: v1.0.0 @@ -32106,65 +31681,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/kevinburke/ssh_config -Version: v0.0.0-20190725054713-01f96b0aa0cd -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/kevinburke/ssh_config@v0.0.0-20190725054713-01f96b0aa0cd/LICENSE: - -Copyright (c) 2017 Kevin Burke. - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -=================== - -The lexer and parser borrow heavily from github.com/pelletier/go-toml. The -license for that project is copied below. - -The MIT License (MIT) - -Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/kisielk/errcheck Version: v1.2.0 @@ -32349,13 +31865,13 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/kr/pty -Version: v1.1.8 +Version: v1.1.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/pty@v1.1.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/kr/pty@v1.1.1/License: -Copyright (c) 2019 Keith Rarick +Copyright (c) 2011 Keith Rarick Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated @@ -33055,37 +32571,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/mitchellh/go-wordwrap -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mitchellh/go-wordwrap@v1.0.0/LICENSE.md: - -The MIT License (MIT) - -Copyright (c) 2014 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/mitchellh/iochan Version: v1.0.0 @@ -35105,14 +34590,6 @@ Contents of probable licence file $GOMODCACHE/github.com/oxtoacart/bpool@v0.0.0- limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/pelletier/go-buffruneio -Version: v0.2.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -No licence file provided. - -------------------------------------------------------------------------------- Dependency : github.com/peterbourgon/diskv Version: v2.0.1+incompatible @@ -36364,44 +35841,6 @@ Contents of probable licence file $GOMODCACHE/github.com/spf13/afero@v1.2.2/LICE of your accepting any such warranty or additional liability. --------------------------------------------------------------------------------- -Dependency : github.com/src-d/gcfg -Version: v1.4.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/src-d/gcfg@v1.4.0/LICENSE: - -Copyright (c) 2012 Péter Surányi. Portions Copyright (c) 2009 The Go -Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/stretchr/objx Version: v0.2.0 @@ -36711,12 +36150,12 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/diag@v0.0.0-202002 -------------------------------------------------------------------------------- -Dependency : github.com/urso/diag-ecs -Version: v0.0.0-20200210114345-ab085841dcb9 +Dependency : github.com/urso/go-bin +Version: v0.0.0-20180220135811-781c575c9f0e Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/diag-ecs@v0.0.0-20200210114345-ab085841dcb9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/urso/go-bin@v0.0.0-20180220135811-781c575c9f0e/LICENSE: Apache License Version 2.0, January 2004 @@ -36922,12 +36361,20 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/diag-ecs@v0.0.0-20 -------------------------------------------------------------------------------- -Dependency : github.com/urso/go-bin -Version: v0.0.0-20180220135811-781c575c9f0e +Dependency : github.com/urso/magetools +Version: v0.0.0-20190919040553-290c89e0c230 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/go-bin@v0.0.0-20180220135811-781c575c9f0e/LICENSE: +No licence file provided. + +-------------------------------------------------------------------------------- +Dependency : github.com/urso/qcgen +Version: v0.0.0-20180131103024-0b059e7db4f4 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180131103024-0b059e7db4f4/LICENSE: Apache License Version 2.0, January 2004 @@ -37133,14 +36580,52 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/go-bin@v0.0.0-2018 -------------------------------------------------------------------------------- -Dependency : github.com/urso/magetools -Version: v0.0.0-20200125210132-c2e338f92f3a +Dependency : github.com/vbatts/tar-split +Version: v0.11.1 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/vbatts/tar-split@v0.11.1/LICENSE: + +Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/xanzy/go-gitlab +Version: v0.22.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/magetools@v0.0.0-20200125210132-c2e338f92f3a/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xanzy/go-gitlab@v0.22.3/LICENSE: - Apache License +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -37320,7 +36805,7 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/magetools@v0.0.0-2 APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -37328,7 +36813,7 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/magetools@v0.0.0-2 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -37343,13 +36828,15 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/magetools@v0.0.0-2 limitations under the License. + -------------------------------------------------------------------------------- -Dependency : github.com/urso/qcgen -Version: v0.0.0-20180131103024-0b059e7db4f4 +Dependency : github.com/xdg/stringprep +Version: v1.0.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180131103024-0b059e7db4f4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xdg/stringprep@v1.0.0/LICENSE: + Apache License Version 2.0, January 2004 @@ -37526,41 +37013,15 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180 incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/urso/sderr -Version: v0.0.0-20200210124243-c2a16f3d43ec +Dependency : github.com/xeipuuv/gojsonpointer +Version: v0.0.0-20190905194746-02993c407bfb Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v0.0.0-20190905194746-02993c407bfb/LICENSE-APACHE-2.0.txt: + Apache License Version 2.0, January 2004 @@ -37750,7 +37211,7 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2015 xeipuuv Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -37766,52 +37227,15 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200 -------------------------------------------------------------------------------- -Dependency : github.com/vbatts/tar-split -Version: v0.11.1 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/vbatts/tar-split@v0.11.1/LICENSE: - -Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/xanzy/go-gitlab -Version: v0.22.3 +Dependency : github.com/xeipuuv/gojsonreference +Version: v0.0.0-20180127040603-bd5ef7bd5415 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xanzy/go-gitlab@v0.22.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonreference@v0.0.0-20180127040603-bd5ef7bd5415/LICENSE-APACHE-2.0.txt: -Apache License + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -37991,7 +37415,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -37999,7 +37423,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2015 xeipuuv Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38014,14 +37438,14 @@ Apache License limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/xanzy/ssh-agent -Version: v0.2.1 +Dependency : github.com/xeipuuv/gojsonschema +Version: v0.0.0-20181112162635-ac52e6811b56 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonschema@v0.0.0-20181112162635-ac52e6811b56/LICENSE-APACHE-2.0.txt: + Apache License Version 2.0, January 2004 @@ -38203,7 +37627,7 @@ Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -38211,7 +37635,7 @@ Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2015 xeipuuv Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38226,199 +37650,77 @@ Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/ limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/xdg/stringprep -Version: v1.0.0 -Licence type (autodetected): Apache-2.0 +Dependency : github.com/yuin/gopher-lua +Version: v0.0.0-20170403160031-b402f3114ec7 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xdg/stringprep@v1.0.0/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +Contents of probable licence file $GOMODCACHE/github.com/yuin/gopher-lua@v0.0.0-20170403160031-b402f3114ec7/LICENSE: - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +The MIT License (MIT) - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +Copyright (c) 2015 Yusuke Inuzuka - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +-------------------------------------------------------------------------------- +Dependency : go.elastic.co/fastjson +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.0.0/LICENSE: - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +Copyright 2018 Elasticsearch BV - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. + http://www.apache.org/licenses/LICENSE-2.0 - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +--- - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +Copyright (c) 2016 Mail.Ru Group - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/xeipuuv/gojsonpointer -Version: v0.0.0-20190905194746-02993c407bfb +Dependency : go.opencensus.io +Version: v0.22.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v0.0.0-20190905194746-02993c407bfb/LICENSE-APACHE-2.0.txt: +Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.22.2/LICENSE: Apache License @@ -38609,7 +37911,7 @@ Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2015 xeipuuv + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38623,1835 +37925,259 @@ Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v See the License for the specific language governing permissions and limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/xeipuuv/gojsonreference -Version: v0.0.0-20180127040603-bd5ef7bd5415 -Licence type (autodetected): Apache-2.0 +Dependency : go.uber.org/goleak +Version: v1.0.0 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonreference@v0.0.0-20180127040603-bd5ef7bd5415/LICENSE-APACHE-2.0.txt: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.0.0/LICENSE: - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +The MIT License (MIT) - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +Copyright (c) 2018 Uber Technologies, Inc. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. +-------------------------------------------------------------------------------- +Dependency : go.uber.org/tools +Version: v0.0.0-20190618225709-2cfd321de3ee +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +Contents of probable licence file $GOMODCACHE/go.uber.org/tools@v0.0.0-20190618225709-2cfd321de3ee/LICENSE: - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +Copyright (c) 2017 Uber Technologies, Inc. - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/exp +Version: v0.0.0-20191227195350-da58074b4299 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20191227195350-da58074b4299/LICENSE: - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +Copyright (c) 2009 The Go Authors. All rights reserved. - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/image +Version: v0.0.0-20190802002840-cff245a6509b +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. +Contents of probable licence file $GOMODCACHE/golang.org/x/image@v0.0.0-20190802002840-cff245a6509b/LICENSE: - Copyright 2015 xeipuuv +Copyright (c) 2009 The Go Authors. All rights reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - http://www.apache.org/licenses/LICENSE-2.0 + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/xeipuuv/gojsonschema -Version: v0.0.0-20181112162635-ac52e6811b56 -Licence type (autodetected): Apache-2.0 +Dependency : golang.org/x/mobile +Version: v0.0.0-20190719004257-d2bd2a29d028 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonschema@v0.0.0-20181112162635-ac52e6811b56/LICENSE-APACHE-2.0.txt: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Contents of probable licence file $GOMODCACHE/golang.org/x/mobile@v0.0.0-20190719004257-d2bd2a29d028/LICENSE: - 1. Definitions. +Copyright (c) 2009 The Go Authors. All rights reserved. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/mod +Version: v0.3.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.3.0/LICENSE: - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +Copyright (c) 2009 The Go Authors. All rights reserved. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/xerrors +Version: v0.0.0-20191204190536-9bdfabe68543 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/LICENSE: - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +Copyright (c) 2019 The Go Authors. All rights reserved. - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +-------------------------------------------------------------------------------- +Dependency : google.golang.org/appengine +Version: v1.6.5 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +Contents of probable licence file $GOMODCACHE/google.golang.org/appengine@v1.6.5/LICENSE: - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2015 xeipuuv - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/yuin/gopher-lua -Version: v0.0.0-20170403160031-b402f3114ec7 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/yuin/gopher-lua@v0.0.0-20170403160031-b402f3114ec7/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 Yusuke Inuzuka - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : go.elastic.co/fastjson -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.0.0/LICENSE: - -Copyright 2018 Elasticsearch BV - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ---- - -Copyright (c) 2016 Mail.Ru Group - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : go.opencensus.io -Version: v0.22.2 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.22.2/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- -Dependency : go.uber.org/goleak -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.0.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2018 Uber Technologies, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : go.uber.org/tools -Version: v0.0.0-20190618225709-2cfd321de3ee -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.uber.org/tools@v0.0.0-20190618225709-2cfd321de3ee/LICENSE: - -Copyright (c) 2017 Uber Technologies, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/exp -Version: v0.0.0-20191227195350-da58074b4299 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20191227195350-da58074b4299/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/image -Version: v0.0.0-20190802002840-cff245a6509b -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/image@v0.0.0-20190802002840-cff245a6509b/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/mobile -Version: v0.0.0-20190719004257-d2bd2a29d028 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/mobile@v0.0.0-20190719004257-d2bd2a29d028/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/mod -Version: v0.3.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.3.0/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/xerrors -Version: v0.0.0-20191204190536-9bdfabe68543 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/LICENSE: - -Copyright (c) 2019 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : google.golang.org/appengine -Version: v1.6.5 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/google.golang.org/appengine@v1.6.5/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : google.golang.org/protobuf -Version: v1.23.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.23.0/LICENSE: - -Copyright (c) 2018 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/airbrake/gobrake.v2 -Version: v2.0.9 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/airbrake/gobrake.v2@v2.0.9/LICENSE: - -Copyright (c) 2014 The Gobrake Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/alecthomas/kingpin.v2 -Version: v2.2.6 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/alecthomas/kingpin.v2@v2.2.6/COPYING: - -Copyright (C) 2014 Alec Thomas - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/check.v1 -Version: v1.0.0-20190902080502-41f04d3bba15 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20190902080502-41f04d3bba15/LICENSE: - -Gocheck - A rich testing framework for Go - -Copyright (c) 2010-2013 Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/errgo.v2 -Version: v2.1.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/errgo.v2@v2.1.0/LICENSE: - -Copyright © 2013, Roger Peppe -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of this project nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/fsnotify.v1 -Version: v1.4.7 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/fsnotify.v1@v1.4.7/LICENSE: - -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2012 fsnotify Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/gemnasium/logrus-airbrake-hook.v2 -Version: v2.1.2 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/gemnasium/logrus-airbrake-hook.v2@v2.1.2/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 Gemnasium - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/aescts.v1 -Version: v1.0.1 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/aescts.v1@v1.0.1/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/dnsutils.v1 -Version: v1.0.1 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/dnsutils.v1@v1.0.1/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/goidentity.v3 -Version: v3.0.0 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/goidentity.v3@v3.0.0/LICENSE: Apache License Version 2.0, January 2004 @@ -40633,7 +38359,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/goidentity.v3@v APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -40641,7 +38367,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/goidentity.v3@v same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -40657,12 +38383,255 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/goidentity.v3@v -------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/rpc.v1 -Version: v1.1.0 +Dependency : google.golang.org/protobuf +Version: v1.23.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.23.0/LICENSE: + +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/airbrake/gobrake.v2 +Version: v2.0.9 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/airbrake/gobrake.v2@v2.0.9/LICENSE: + +Copyright (c) 2014 The Gobrake Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/alecthomas/kingpin.v2 +Version: v2.2.6 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/alecthomas/kingpin.v2@v2.2.6/COPYING: + +Copyright (C) 2014 Alec Thomas + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/check.v1 +Version: v1.0.0-20190902080502-41f04d3bba15 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20190902080502-41f04d3bba15/LICENSE: + +Gocheck - A rich testing framework for Go + +Copyright (c) 2010-2013 Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/errgo.v2 +Version: v2.1.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/errgo.v2@v2.1.0/LICENSE: + +Copyright © 2013, Roger Peppe +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of this project nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/fsnotify.v1 +Version: v1.4.7 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/fsnotify.v1@v1.4.7/LICENSE: + +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2012 fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/gemnasium/logrus-airbrake-hook.v2 +Version: v2.1.2 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/gemnasium/logrus-airbrake-hook.v2@v2.1.2/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 Gemnasium + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/jcmturner/aescts.v1 +Version: v1.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/aescts.v1@v1.0.1/LICENSE: Apache License Version 2.0, January 2004 @@ -40844,7 +38813,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -40852,7 +38821,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -40868,12 +38837,12 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L -------------------------------------------------------------------------------- -Dependency : gopkg.in/src-d/go-billy.v4 -Version: v4.3.2 +Dependency : gopkg.in/jcmturner/dnsutils.v1 +Version: v1.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/dnsutils.v1@v1.0.1/LICENSE: Apache License Version 2.0, January 2004 @@ -41055,7 +39024,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -41063,7 +39032,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2017 Sourced Technologies S.L. + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41079,12 +39048,12 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/ -------------------------------------------------------------------------------- -Dependency : gopkg.in/src-d/go-git-fixtures.v3 -Version: v3.5.0 +Dependency : gopkg.in/jcmturner/goidentity.v3 +Version: v3.0.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git-fixtures.v3@v3.5.0/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/goidentity.v3@v3.0.0/LICENSE: Apache License Version 2.0, January 2004 @@ -41274,7 +39243,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git-fixtures.v3@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2017 Sourced Technologies, S.L. + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41290,12 +39259,12 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git-fixtures.v3@ -------------------------------------------------------------------------------- -Dependency : gopkg.in/src-d/go-git.v4 -Version: v4.13.1 +Dependency : gopkg.in/jcmturner/rpc.v1 +Version: v1.1.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git.v4@v4.13.1/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/LICENSE: Apache License Version 2.0, January 2004 @@ -41477,7 +39446,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git.v4@v4.13.1/L APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -41485,7 +39454,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git.v4@v4.13.1/L same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018 Sourced Technologies, S.L. + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41539,40 +39508,6 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : gopkg.in/warnings.v0 -Version: v0.1.2 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/warnings.v0@v0.1.2/LICENSE: - -Copyright (c) 2016 Péter Surányi. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v3 Version: v3.0.0-20200313102051-9f266ea9e77c diff --git a/README.md b/README.md index 63501084508..1b383c288de 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ your dev environment to build Beats from the source. ## Snapshots -For testing purposes, we generate snapshot builds that you can find [here](https://beats-ci.elastic.co/job/elastic+beats+master+multijob-package-linux/lastSuccessfulBuild/gcsObjects/). Please be aware that these are built on top of master and are not meant for production. +For testing purposes, we generate snapshot builds that you can find [here](https://beats-ci.elastic.co/job/Beats/job/packaging/job/master/lastSuccessfulBuild/gcsObjects/). Please be aware that these are built on top of master and are not meant for production. ## CI diff --git a/deploy/kubernetes/auditbeat-kubernetes.yaml b/deploy/kubernetes/auditbeat-kubernetes.yaml index bd6c4736c8b..24492c50653 100644 --- a/deploy/kubernetes/auditbeat-kubernetes.yaml +++ b/deploy/kubernetes/auditbeat-kubernetes.yaml @@ -196,14 +196,15 @@ spec: path: /etc - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-daemonset-modules - name: data hostPath: + # When auditbeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/auditbeat-data type: DirectoryOrCreate - name: run-containerd diff --git a/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml b/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml index 21ffb167107..39eaf726eef 100644 --- a/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml +++ b/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml @@ -109,14 +109,15 @@ spec: path: /etc - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-daemonset-modules - name: data hostPath: + # When auditbeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/auditbeat-data type: DirectoryOrCreate - name: run-containerd diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml index 49e4ff12f0b..7399635e42d 100644 --- a/deploy/kubernetes/filebeat-kubernetes.yaml +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -112,7 +112,7 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: filebeat-config - name: varlibdockercontainers hostPath: @@ -123,6 +123,7 @@ spec: # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/filebeat-data type: DirectoryOrCreate --- diff --git a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml index 20c742d518d..b6df8f31fdb 100644 --- a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml +++ b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml @@ -68,7 +68,7 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: filebeat-config - name: varlibdockercontainers hostPath: @@ -79,5 +79,6 @@ spec: # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/filebeat-data type: DirectoryOrCreate diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 216bbc24f9f..9690b9cb84f 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -177,14 +177,15 @@ spec: path: /var/run/docker.sock - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-modules - name: data hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) path: /var/lib/metricbeat-data type: DirectoryOrCreate --- @@ -302,11 +303,11 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-modules --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml index 96f841c4519..0197fe136b6 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml @@ -84,13 +84,14 @@ spec: path: /var/run/docker.sock - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-modules - name: data hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) path: /var/lib/metricbeat-data type: DirectoryOrCreate diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml index 8b0c5351ed0..0e11187cac3 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml @@ -61,9 +61,9 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-modules diff --git a/dev-tools/mage/dashboard.go b/dev-tools/mage/dashboard.go index 562a3bf8f79..764a9e28695 100644 --- a/dev-tools/mage/dashboard.go +++ b/dev-tools/mage/dashboard.go @@ -44,7 +44,7 @@ func ExportDashboard() error { return err } - dashboardCmd := sh.RunCmd("go", "run", "-mod", "vendor", filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go")) + dashboardCmd := sh.RunCmd("go", "run", filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go")) // TODO: This is currently hardcoded for KB 7, we need to figure out what we do for KB 8 if applicable file := CWD("module", module, "_meta/kibana/7/dashboard", id+".json") diff --git a/dev-tools/mage/fmt.go b/dev-tools/mage/fmt.go index 02a8001895e..b4463696b81 100644 --- a/dev-tools/mage/fmt.go +++ b/dev-tools/mage/fmt.go @@ -74,8 +74,8 @@ func GoImports() error { return err } } else { - if err := gotool.Get( - gotool.Get.Package(filepath.Join(GoImportsImportPath)), + if err := gotool.Install( + gotool.Install.Package(filepath.Join(GoImportsImportPath)), ); err != nil { return err } diff --git a/dev-tools/notice/overrides.json b/dev-tools/notice/overrides.json index c40ec350c70..16c8447a13d 100644 --- a/dev-tools/notice/overrides.json +++ b/dev-tools/notice/overrides.json @@ -9,3 +9,4 @@ {"name": "github.com/chzyer/logex", "licenceType": "MIT"} {"name": "github.com/munnerz/goautoneg", "licenceType": "BSD-3-Clause"} {"name": "github.com/pelletier/go-buffruneio", "licenceType": "MIT"} +{"name": "github.com/urso/magetools", "licenceType": "Apache-2.0"} diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 1123bb14f7b..9080b7c534d 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -30,7 +30,7 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint RUN groupadd --gid 1000 {{ .BeatName }} RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ - chown -R root:{{ .BeatName }} {{ $beatHome }} && \ + chown -R root:root {{ $beatHome }} && \ find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ chmod 0750 {{ $beatBinary }} && \ @@ -43,7 +43,7 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs {{- if ne .user "root" }} -RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }} +RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} {{- end }} USER {{ .user }} diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index a695efaab3e..6e8112eb268 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2040,6 +2041,47 @@ type: keyword The type of traffic: IPv4, IPv6, or EFA. +type: keyword + +-- + +[[exported-fields-awscloudwatch]] +== awscloudwatch fields + +Fields from AWS CloudWatch logs. + + + +[float] +=== awscloudwatch + +Fields from AWS CloudWatch logs. + + + +*`awscloudwatch.log_group`*:: ++ +-- +The name of the log group to which this event belongs. + +type: keyword + +-- + +*`awscloudwatch.log_stream`*:: ++ +-- +The name of the log stream to which this event belongs. + +type: keyword + +-- + +*`awscloudwatch.ingestion_time`*:: ++ +-- +The time the event was ingested in AWS CloudWatch. + type: keyword -- @@ -24755,6 +24797,16 @@ type: keyword Java class the log is coming from. +type: keyword + +-- + +*`kafka.log.thread`*:: ++ +-- +Thread name the log is coming from. + + type: keyword -- @@ -34544,6 +34596,36 @@ type: keyword -- + +*`suricata.eve.tls.ja3s.string`*:: ++ +-- +type: keyword + +-- + +*`suricata.eve.tls.ja3s.hash`*:: ++ +-- +type: keyword + +-- + + +*`suricata.eve.tls.ja3.string`*:: ++ +-- +type: keyword + +-- + +*`suricata.eve.tls.ja3.hash`*:: ++ +-- +type: keyword + +-- + *`suricata.eve.app_proto_ts`*:: + -- diff --git a/filebeat/docs/modules/kafka.asciidoc b/filebeat/docs/modules/kafka.asciidoc index d9319b43b50..5e2145a348e 100644 --- a/filebeat/docs/modules/kafka.asciidoc +++ b/filebeat/docs/modules/kafka.asciidoc @@ -11,6 +11,8 @@ This file is generated! See scripts/docs_collector.py The +{modulename}+ module collects and parses the logs created by https://kafka.apache.org/[Kafka]. +The module has additional support for parsing thread ID from logs. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/filebeat/input/file/state.go b/filebeat/input/file/state.go index dde3c6c5421..ef255243b4c 100644 --- a/filebeat/input/file/state.go +++ b/filebeat/input/file/state.go @@ -30,16 +30,16 @@ import ( // State is used to communicate the reading state of a file type State struct { - Id string `json:"-"` // local unique id to make comparison more efficient - Finished bool `json:"-"` // harvester state - Fileinfo os.FileInfo `json:"-"` // the file info - Source string `json:"source"` - Offset int64 `json:"offset"` - Timestamp time.Time `json:"timestamp"` - TTL time.Duration `json:"ttl"` - Type string `json:"type"` - Meta map[string]string `json:"meta"` - FileStateOS file.StateOS + Id string `json:"-" struct:"-"` // local unique id to make comparison more efficient + Finished bool `json:"-" struct:"-"` // harvester state + Fileinfo os.FileInfo `json:"-" struct:"-"` // the file info + Source string `json:"source" struct:"source"` + Offset int64 `json:"offset" struct:"offset"` + Timestamp time.Time `json:"timestamp" struct:"timestamp"` + TTL time.Duration `json:"ttl" struct:"ttl"` + Type string `json:"type" struct:"type"` + Meta map[string]string `json:"meta" struct:"meta,omitempty"` + FileStateOS file.StateOS `json:"FileStateOS" struct:"FileStateOS"` } // NewState creates a new file state diff --git a/filebeat/input/file/states.go b/filebeat/input/file/states.go index fc50dd904c0..34704b41dba 100644 --- a/filebeat/input/file/states.go +++ b/filebeat/input/file/states.go @@ -94,6 +94,12 @@ func (s *States) findPrevious(id string) int { // The number of states that were cleaned up and number of states that can be // cleaned up in the future is returned. func (s *States) Cleanup() (int, int) { + return s.CleanupWith(nil) +} + +// CleanupWith cleans up the state array. It calls `fn` with the state ID, for +// each entry to be removed. +func (s *States) CleanupWith(fn func(string)) (int, int) { s.Lock() defer s.Unlock() @@ -114,7 +120,11 @@ func (s *States) Cleanup() (int, int) { continue } - delete(s.idx, state.ID()) + id := state.ID() + delete(s.idx, id) + if fn != nil { + fn(id) + } logp.Debug("state", "State removed for %v because of older: %v", state.Source, state.TTL) L-- diff --git a/filebeat/input/v2/input-cursor/clean.go b/filebeat/input/v2/input-cursor/clean.go index e833a9eead0..e4daaaf6e7d 100644 --- a/filebeat/input/v2/input-cursor/clean.go +++ b/filebeat/input/v2/input-cursor/clean.go @@ -20,8 +20,10 @@ package cursor import ( "time" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/timed" "github.com/elastic/go-concert/unison" + + "github.com/elastic/beats/v7/libbeat/logp" ) // cleaner removes finished entries from the registry file. @@ -41,5 +43,81 @@ type cleaner struct { // for a long time, and the life time has been exhausted, then the resource will be removed immediately // once the last event has been ACKed. func (c *cleaner) run(canceler unison.Canceler, store *store, interval time.Duration) { - panic("TODO: implement me") + started := time.Now() + timed.Periodic(canceler, interval, func() { + gcStore(c.log, started, store) + }) +} + +// gcStore looks for resources to remove and deletes these. `gcStore` receives +// the start timestamp of the cleaner as reference. If we have entries without +// updates in the registry, that are older than `started`, we will use `started +// + ttl` to decide if an entry will be removed. This way old entries are not +// removed immediately on startup if the Beat is down for a longer period of +// time. +func gcStore(log *logp.Logger, started time.Time, store *store) { + log.Debugf("Start store cleanup") + defer log.Debugf("Done store cleanup") + + states := store.ephemeralStore + states.mu.Lock() + defer states.mu.Unlock() + + keys := gcFind(states.table, started, time.Now()) + if len(keys) == 0 { + log.Debug("No entries to remove were found") + return + } + + if err := gcClean(store, keys); err != nil { + log.Errorf("Failed to remove all entries from the registry: %+v", err) + } +} + +// gcFind searches the store of resources that can be removed. A set of keys to delete is returned. +func gcFind(table map[string]*resource, started, now time.Time) map[string]struct{} { + keys := map[string]struct{}{} + for key, resource := range table { + clean := checkCleanResource(started, now, resource) + if !clean { + // do not clean the resource if it is still live or not serialized to the persistent store yet. + continue + } + keys[key] = struct{}{} + } + + return keys +} + +// gcClean removes key value pairs in the removeSet from the store. +// If deletion in the persistent store fails the entry is kept in memory and +// eventually cleaned up later. +func gcClean(store *store, removeSet map[string]struct{}) error { + for key := range removeSet { + if err := store.persistentStore.Remove(key); err != nil { + return err + } + delete(store.ephemeralStore.table, key) + } + return nil +} + +// checkCleanResource returns true for a key-value pair is assumed to be old, +// if is not in use and there are no more pending updates that still need to be +// written to the persistent store anymore. +func checkCleanResource(started, now time.Time, resource *resource) bool { + if !resource.Finished() { + return false + } + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + ttl := resource.internalState.TTL + reference := resource.internalState.Updated + if started.After(reference) { + reference = started + } + + return reference.Add(ttl).Before(now) && resource.stored } diff --git a/filebeat/input/v2/input-cursor/clean_test.go b/filebeat/input/v2/input-cursor/clean_test.go new file mode 100644 index 00000000000..452120f0358 --- /dev/null +++ b/filebeat/input/v2/input-cursor/clean_test.go @@ -0,0 +1,162 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestGCStore(t *testing.T) { + t.Run("empty store", func(t *testing.T) { + started := time.Now() + + backend := createSampleStore(t, nil) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("state is still alive", func(t *testing.T) { + started := time.Now() + const ttl = 60 * time.Second + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl / 2), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + checkEqualStoreState(t, initState, backend.snapshot()) + }) + + t.Run("old state can be removed", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("old state is not removed if cleanup is not active long enough", func(t *testing.T) { + const ttl = 60 * time.Minute + started := time.Now() + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-2 * ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + checkEqualStoreState(t, initState, backend.snapshot()) + }) + + t.Run("old state but resource is accessed", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + // access resource and check it is not gc'ed + res := store.Get("test::key") + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, initState, backend.snapshot()) + + // release resource and check it gets gc'ed + res.Release() + want := map[string]state{} + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("old state but resource has pending updates", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + // create pending update operation + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + res.Release() + + // cleanup fails + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, initState, backend.snapshot()) + + // cancel operation (no more pending operations) and try to gc again + op.done(1) + gcStore(logp.NewLogger("test"), started, store) + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) +} diff --git a/filebeat/input/v2/input-cursor/cursor_test.go b/filebeat/input/v2/input-cursor/cursor_test.go new file mode 100644 index 00000000000..ed1f4120869 --- /dev/null +++ b/filebeat/input/v2/input-cursor/cursor_test.go @@ -0,0 +1,124 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCursor_IsNew(t *testing.T) { + t.Run("true if key is not in store", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.True(t, cursor.IsNew()) + }) + + t.Run("true if key is in store but without cursor value", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: nil}, + })) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.True(t, cursor.IsNew()) + }) + + t.Run("false if key with cursor value is in persistent store", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.False(t, cursor.IsNew()) + }) + + t.Run("false if key with cursor value is in memory store only", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: nil}, + })) + defer store.Release() + + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + cursor := makeCursor(store, res) + require.False(t, cursor.IsNew()) + }) +} + +func TestCursor_Unpack(t *testing.T) { + t.Run("nothing to unpack if key is new", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "", st) + }) + + t.Run("unpack fails if types are not compatible", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + var st struct{ A uint } + cursor := makeCursor(store, store.Get("test::key")) + require.Error(t, cursor.Unpack(&st)) + }) + + t.Run("unpack from state in persistent store", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "test", st) + }) + + t.Run("unpack from in memory state if updates are pending", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "test-state-update", st) + }) +} diff --git a/filebeat/input/v2/input-cursor/input.go b/filebeat/input/v2/input-cursor/input.go index 94faeeaadda..a768823d9f6 100644 --- a/filebeat/input/v2/input-cursor/input.go +++ b/filebeat/input/v2/input-cursor/input.go @@ -18,11 +18,19 @@ package cursor import ( + "context" "fmt" + "runtime/debug" "time" + "github.com/urso/sderr" + + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/unison" + input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/logp" ) // Input interface for cursor based inputs. This interface must be implemented @@ -62,7 +70,29 @@ func (inp *managedInput) Name() string { return inp.input.Name() } // Test runs the Test method for each configured source. func (inp *managedInput) Test(ctx input.TestContext) error { - panic("TODO: implement me") + var grp unison.MultiErrGroup + for _, source := range inp.sources { + source := source + grp.Go(func() (err error) { + return inp.testSource(ctx, source) + }) + } + + errs := grp.Wait() + if len(errs) > 0 { + return sderr.WrapAll(errs, "input tests failed") + } + return nil +} + +func (inp *managedInput) testSource(ctx input.TestContext, source Source) (err error) { + defer func() { + if v := recover(); v != nil { + err = fmt.Errorf("input panic with: %+v\n%s", v, debug.Stack()) + ctx.Logger.Errorf("Input crashed with: %+v", err) + } + }() + return inp.input.Test(source, ctx) } // Run creates a go-routine per source, waiting until all go-routines have @@ -73,7 +103,68 @@ func (inp *managedInput) Run( ctx input.Context, pipeline beat.PipelineConnector, ) (err error) { - panic("TODO: implement me") + // Setup cancellation using a custom cancel context. All workers will be + // stopped if one failed badly by returning an error. + cancelCtx, cancel := context.WithCancel(ctxtool.FromCanceller(ctx.Cancelation)) + defer cancel() + ctx.Cancelation = cancelCtx + + var grp unison.MultiErrGroup + for _, source := range inp.sources { + source := source + grp.Go(func() (err error) { + // refine per worker context + inpCtx := ctx + inpCtx.ID = ctx.ID + "::" + source.Name() + inpCtx.Logger = ctx.Logger.With("source", source.Name()) + + if err = inp.runSource(inpCtx, inp.manager.store, source, pipeline); err != nil { + cancel() + } + return err + }) + } + + if errs := grp.Wait(); len(errs) > 0 { + return sderr.WrapAll(errs, "input %{id} failed", ctx.ID) + } + return nil +} + +func (inp *managedInput) runSource( + ctx input.Context, + store *store, + source Source, + pipeline beat.PipelineConnector, +) (err error) { + defer func() { + if v := recover(); v != nil { + err = fmt.Errorf("input panic with: %+v\n%s", v, debug.Stack()) + ctx.Logger.Errorf("Input crashed with: %+v", err) + } + }() + + client, err := pipeline.ConnectWith(beat.ClientConfig{ + CloseRef: ctx.Cancelation, + ACKEvents: newInputACKHandler(ctx.Logger), + }) + if err != nil { + return err + } + defer client.Close() + + resourceKey := inp.createSourceID(source) + resource, err := inp.manager.lock(ctx, resourceKey) + if err != nil { + return err + } + defer releaseResource(resource) + + store.UpdateTTL(resource, inp.cleanTimeout) + + cursor := makeCursor(store, resource) + publisher := &cursorPublisher{canceler: ctx.Cancelation, client: client, cursor: &cursor} + return inp.input.Run(ctx, source, cursor, publisher) } func (inp *managedInput) createSourceID(s Source) string { @@ -82,3 +173,28 @@ func (inp *managedInput) createSourceID(s Source) string { } return fmt.Sprintf("%v::%v", inp.manager.Type, s.Name()) } + +func newInputACKHandler(log *logp.Logger) func([]interface{}) { + return func(private []interface{}) { + var n uint + var last int + for i := 0; i < len(private); i++ { + current := private[i] + if current == nil { + continue + } + + if _, ok := current.(*updateOp); !ok { + continue + } + + n++ + last = i + } + + if n == 0 { + return + } + private[last].(*updateOp).Execute(n) + } +} diff --git a/filebeat/input/v2/input-cursor/manager.go b/filebeat/input/v2/input-cursor/manager.go index ee1b2bc7939..2a4310dc778 100644 --- a/filebeat/input/v2/input-cursor/manager.go +++ b/filebeat/input/v2/input-cursor/manager.go @@ -18,15 +18,19 @@ package cursor import ( + "errors" + "sync" "time" + "github.com/urso/sderr" + + "github.com/elastic/go-concert/unison" + input "github.com/elastic/beats/v7/filebeat/input/v2" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore" - - "github.com/elastic/go-concert/unison" ) // InputManager is used to create, manage, and coordinate stateful inputs and @@ -60,7 +64,9 @@ type InputManager struct { // that will be used to collect events from each source. Configure func(cfg *common.Config) ([]Source, Input, error) - store *store + initOnce sync.Once + initErr error + store *store } // Source describe a source the input can collect data from. @@ -70,22 +76,118 @@ type Source interface { Name() string } +var errNoSourceConfigured = errors.New("no source has been configured") +var errNoInputRunner = errors.New("no input runner available") + // StateStore interface and configurations used to give the Manager access to the persistent store. type StateStore interface { Access() (*statestore.Store, error) CleanupInterval() time.Duration } +func (cim *InputManager) init() error { + cim.initOnce.Do(func() { + if cim.DefaultCleanTimeout <= 0 { + cim.DefaultCleanTimeout = 30 * time.Minute + } + + log := cim.Logger.With("input_type", cim.Type) + var store *store + store, cim.initErr = openStore(log, cim.StateStore, cim.Type) + if cim.initErr != nil { + return + } + + cim.store = store + }) + + return cim.initErr +} + // Init starts background processes for deleting old entries from the // persistent store if mode is ModeRun. func (cim *InputManager) Init(group unison.Group, mode v2.Mode) error { - panic("TODO: implement me") + if mode != v2.ModeRun { + return nil + } + + if err := cim.init(); err != nil { + return err + } + + log := cim.Logger.With("input_type", cim.Type) + + store := cim.store + cleaner := &cleaner{log: log} + store.Retain() + err := group.Go(func(canceler unison.Canceler) error { + defer cim.shutdown() + defer store.Release() + interval := cim.StateStore.CleanupInterval() + if interval <= 0 { + interval = 5 * time.Minute + } + cleaner.run(canceler, store, interval) + return nil + }) + if err != nil { + store.Release() + cim.shutdown() + return sderr.Wrap(err, "Can not start registry cleanup process") + } + + return nil +} + +func (cim *InputManager) shutdown() { + cim.store.Release() } // Create builds a new v2.Input using the provided Configure function. // The Input will run a go-routine per source that has been configured. func (cim *InputManager) Create(config *common.Config) (input.Input, error) { - panic("TODO: implement me") + if err := cim.init(); err != nil { + return nil, err + } + + settings := struct { + ID string `config:"id"` + CleanTimeout time.Duration `config:"clean_timeout"` + }{ID: "", CleanTimeout: cim.DefaultCleanTimeout} + if err := config.Unpack(&settings); err != nil { + return nil, err + } + + sources, inp, err := cim.Configure(config) + if err != nil { + return nil, err + } + if len(sources) == 0 { + return nil, errNoSourceConfigured + } + if inp == nil { + return nil, errNoInputRunner + } + + return &managedInput{ + manager: cim, + userID: settings.ID, + sources: sources, + input: inp, + cleanTimeout: settings.CleanTimeout, + }, nil +} + +// Lock locks a key for exclusive access and returns an resource that can be used to modify +// the cursor state and unlock the key. +func (cim *InputManager) lock(ctx input.Context, key string) (*resource, error) { + resource := cim.store.Get(key) + err := lockResource(ctx.Logger, resource, ctx.Cancelation) + if err != nil { + resource.Release() + return nil, err + } + return resource, nil } func lockResource(log *logp.Logger, resource *resource, canceler input.Canceler) error { diff --git a/filebeat/input/v2/input-cursor/manager_test.go b/filebeat/input/v2/input-cursor/manager_test.go new file mode 100644 index 00000000000..8c531f48f84 --- /dev/null +++ b/filebeat/input/v2/input-cursor/manager_test.go @@ -0,0 +1,607 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "errors" + "fmt" + "runtime" + "sort" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" + "github.com/elastic/beats/v7/libbeat/tests/resources" + "github.com/elastic/go-concert/unison" +) + +type fakeTestInput struct { + OnTest func(Source, input.TestContext) error + OnRun func(input.Context, Source, Cursor, Publisher) error +} + +type stringSource string + +func TestManager_Init(t *testing.T) { + // Integration style tests for the InputManager and the state garbage collector + + t.Run("stopping the taskgroup kills internal go-routines", func(t *testing.T) { + numRoutines := runtime.NumGoroutine() + + var grp unison.TaskGroup + store := createSampleStore(t, nil) + manager := &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: store, + Type: "test", + DefaultCleanTimeout: 10 * time.Millisecond, + } + + err := manager.Init(&grp, v2.ModeRun) + require.NoError(t, err) + + time.Sleep(200 * time.Millisecond) + grp.Stop() + + // wait for all go-routines to be gone + + for numRoutines < runtime.NumGoroutine() { + time.Sleep(1 * time.Millisecond) + } + }) + + t.Run("collect old entries after startup", func(t *testing.T) { + store := createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Millisecond, + Updated: time.Now().Add(-24 * time.Hour), + }, + }) + store.GCPeriod = 10 * time.Millisecond + + var grp unison.TaskGroup + defer grp.Stop() + manager := &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: store, + Type: "test", + DefaultCleanTimeout: 10 * time.Millisecond, + } + + err := manager.Init(&grp, v2.ModeRun) + require.NoError(t, err) + + for len(store.snapshot()) > 0 { + time.Sleep(1 * time.Millisecond) + } + }) +} + +func TestManager_Create(t *testing.T) { + t.Run("fail if no source is configured", func(t *testing.T) { + manager := constInput(t, nil, &fakeTestInput{}) + _, err := manager.Create(common.NewConfig()) + require.Error(t, err) + }) + + t.Run("fail if config error", func(t *testing.T) { + manager := failingManager(t, errors.New("oops")) + _, err := manager.Create(common.NewConfig()) + require.Error(t, err) + }) + + t.Run("fail if no input runner is returned", func(t *testing.T) { + manager := constInput(t, sourceList("test"), nil) + _, err := manager.Create(common.NewConfig()) + require.Error(t, err) + }) + + t.Run("configure ok", func(t *testing.T) { + manager := constInput(t, sourceList("test"), &fakeTestInput{}) + _, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + }) + + t.Run("configuring inputs with overlapping sources is allowed", func(t *testing.T) { + manager := simpleManagerWithConfigure(t, func(cfg *common.Config) ([]Source, Input, error) { + config := struct{ Sources []string }{} + err := cfg.Unpack(&config) + return sourceList(config.Sources...), &fakeTestInput{}, err + }) + + _, err := manager.Create(common.MustNewConfigFrom(map[string]interface{}{ + "sources": []string{"a"}, + })) + require.NoError(t, err) + + _, err = manager.Create(common.MustNewConfigFrom(map[string]interface{}{ + "sources": []string{"a"}, + })) + require.NoError(t, err) + }) +} + +func TestManager_InputsTest(t *testing.T) { + var mu sync.Mutex + var seen []string + + sources := sourceList("source1", "source2") + + t.Run("test is run for each source", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ v2.TestContext) error { + mu.Lock() + defer mu.Unlock() + seen = append(seen, source.Name()) + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + err = inp.Test(input.TestContext{}) + require.NoError(t, err) + + sort.Strings(seen) + require.Equal(t, []string{"source1", "source2"}, seen) + }) + + t.Run("cancel gets distributed to all source tests", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(_ Source, ctx v2.TestContext) error { + <-ctx.Cancelation.Done() + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.TODO()) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{Cancelation: ctx}) + }() + + cancel() + wg.Wait() + require.NoError(t, err) + }) + + t.Run("fail if test for one source fails", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + failing := Source(stringSource("source1")) + sources := []Source{failing, stringSource("source2")} + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ v2.TestContext) error { + if source == failing { + t.Log("return error") + return errors.New("oops") + } + t.Log("return ok") + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{}) + t.Logf("Test returned: %v", err) + }() + + wg.Wait() + require.Error(t, err) + }) + + t.Run("panic is captured", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ v2.TestContext) error { + panic("oops") + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{Logger: logp.NewLogger("test")}) + t.Logf("Test returned: %v", err) + }() + + wg.Wait() + require.Error(t, err) + }) +} + +func TestManager_InputsRun(t *testing.T) { + // Integration style tests for the InputManager and Input.Run + + t.Run("input returned with error", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, _ Cursor, _ Publisher) error { + return errors.New("oops") + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + require.Error(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("panic is captured", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, _ Cursor, _ Publisher) error { + panic("oops") + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + require.Error(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("shutdown on signal", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(ctx input.Context, _ Source, _ Cursor, _ Publisher) error { + <-ctx.Cancelation.Done() + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + }() + + cancel() + require.NoError(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("continue sending from last known position", func(t *testing.T) { + log := logp.NewLogger("test") + + type runConfig struct{ Max int } + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + manager := simpleManagerWithConfigure(t, func(cfg *common.Config) ([]Source, Input, error) { + config := runConfig{} + if err := cfg.Unpack(&config); err != nil { + return nil, nil, err + } + + inp := &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, cursor Cursor, pub Publisher) error { + state := struct{ N int }{} + if !cursor.IsNew() { + if err := cursor.Unpack(&state); err != nil { + return fmt.Errorf("failed to unpack cursor: %w", err) + } + } + + for i := 0; i < config.Max; i++ { + event := beat.Event{Fields: common.MapStr{"n": state.N}} + state.N++ + pub.Publish(event, state) + } + return nil + }, + } + + return sourceList("test"), inp, nil + }) + + var ids []int + pipeline := pubtest.ConstClient(&pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { + id := event.Fields["n"].(int) + ids = append(ids, id) + }, + }) + + // create and run first instance + inp, err := manager.Create(common.MustNewConfigFrom(runConfig{Max: 3})) + require.NoError(t, err) + require.NoError(t, inp.Run(input.Context{ + Logger: log, + Cancelation: context.Background(), + }, pipeline)) + + // create and run second instance instance + inp, err = manager.Create(common.MustNewConfigFrom(runConfig{Max: 3})) + require.NoError(t, err) + inp.Run(input.Context{ + Logger: log, + Cancelation: context.Background(), + }, pipeline) + + // verify + assert.Equal(t, []int{0, 1, 2, 3, 4, 5}, ids) + }) + + t.Run("event ACK triggers execution of update operations", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + store := createSampleStore(t, nil) + var wgSend sync.WaitGroup + wgSend.Add(1) + manager := constInput(t, sourceList("key"), &fakeTestInput{ + OnRun: func(ctx input.Context, _ Source, _ Cursor, pub Publisher) error { + defer wgSend.Done() + fields := common.MapStr{"hello": "world"} + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state1") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state2") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state3") + pub.Publish(beat.Event{Fields: fields}, nil) + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state4") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state5") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state6") + return nil + }, + }) + manager.StateStore = store + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // setup publishing pipeline and capture ACKer, so we can simulate progress in the Output + var acker func([]interface{}) + var activeEventPrivate []interface{} + + ackEvents := func(n int) { + data, rest := activeEventPrivate[:n], activeEventPrivate[n:] + activeEventPrivate = rest + acker(data) + } + + var wgACKer sync.WaitGroup + wgACKer.Add(1) + pipeline := &pubtest.FakeConnector{ + ConnectFunc: func(cfg beat.ClientConfig) (beat.Client, error) { + defer wgACKer.Done() + acker = cfg.ACKEvents + return &pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { + activeEventPrivate = append(activeEventPrivate, event.Private) + }, + }, nil + }, + } + + // start the input + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, pipeline) + }() + // wait for test setup to shutdown + defer wg.Wait() + + // wait for setup complete and events being send (pending operations in the pipeline) + wgACKer.Wait() + wgSend.Wait() + + // 1. No cursor state in store yet, all operations are still pending + require.Equal(t, nil, store.snapshot()["test::key"].Cursor) + + // ACK first 2 events and check snapshot state + ackEvents(2) + require.Equal(t, "test-cursor-state2", store.snapshot()["test::key"].Cursor) + + // ACK 1 events and check snapshot state (3 events published) + ackEvents(1) + require.Equal(t, "test-cursor-state3", store.snapshot()["test::key"].Cursor) + + // ACK event without cursor update and check snapshot state not modified + ackEvents(1) + require.Equal(t, "test-cursor-state3", store.snapshot()["test::key"].Cursor) + + // ACK rest + ackEvents(3) + require.Equal(t, "test-cursor-state6", store.snapshot()["test::key"].Cursor) + }) +} + +func TestLockResource(t *testing.T) { + t.Run("can lock unused resource", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + err := lockResource(logp.NewLogger("test"), res, context.TODO()) + require.NoError(t, err) + }) + + t.Run("fail to lock resource in use when context is cancelled", func(t *testing.T) { + log := logp.NewLogger("test") + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + resUsed := store.Get("test::key") + err := lockResource(log, resUsed, context.TODO()) + require.NoError(t, err) + + // fail to lock resource in use + ctx, cancel := context.WithCancel(context.TODO()) + cancel() + resFail := store.Get("test::key") + err = lockResource(log, resFail, ctx) + require.Error(t, err) + resFail.Release() + + // unlock and release resource in use -> it should be marked finished now + releaseResource(resUsed) + require.True(t, resUsed.Finished()) + }) + + t.Run("succeed to lock resource after it has been released", func(t *testing.T) { + log := logp.NewLogger("test") + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + resUsed := store.Get("test::key") + err := lockResource(log, resUsed, context.TODO()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + resOther := store.Get("test::key") + err := lockResource(log, resOther, context.TODO()) + if err == nil { + releaseResource(resOther) + } + }() + + go func() { + time.Sleep(100 * time.Millisecond) + releaseResource(resUsed) + }() + + wg.Wait() // <- block forever if waiting go-routine can not acquire lock + }) +} + +func (s stringSource) Name() string { return string(s) } + +func simpleManagerWithConfigure(t *testing.T, configure func(*common.Config) ([]Source, Input, error)) *InputManager { + return &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: createSampleStore(t, nil), + Type: "test", + Configure: configure, + } +} + +func constConfigureResult(t *testing.T, sources []Source, inp Input, err error) *InputManager { + return simpleManagerWithConfigure(t, func(cfg *common.Config) ([]Source, Input, error) { + return sources, inp, err + }) +} + +func failingManager(t *testing.T, err error) *InputManager { + return constConfigureResult(t, nil, nil, err) +} + +func constInput(t *testing.T, sources []Source, inp Input) *InputManager { + return constConfigureResult(t, sources, inp, nil) +} + +func (f *fakeTestInput) Name() string { return "test" } + +func (f *fakeTestInput) Test(source Source, ctx input.TestContext) error { + if f.OnTest != nil { + return f.OnTest(source, ctx) + } + return nil +} + +func (f *fakeTestInput) Run(ctx input.Context, source Source, cursor Cursor, pub Publisher) error { + if f.OnRun != nil { + return f.OnRun(ctx, source, cursor, pub) + } + return nil +} + +func sourceList(names ...string) []Source { + tmp := make([]Source, len(names)) + for i, name := range names { + tmp[i] = stringSource(name) + } + return tmp +} diff --git a/filebeat/input/v2/input-cursor/publish.go b/filebeat/input/v2/input-cursor/publish.go index 437c4e07373..64096951132 100644 --- a/filebeat/input/v2/input-cursor/publish.go +++ b/filebeat/input/v2/input-cursor/publish.go @@ -22,6 +22,8 @@ import ( input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" + "github.com/elastic/beats/v7/libbeat/statestore" ) // Publisher is used to publish an event and update the cursor in a single call to Publish. @@ -63,10 +65,89 @@ type updateOp struct { // The ACK ordering in the publisher pipeline guarantees that update operations // will be ACKed and executed in the correct order. func (c *cursorPublisher) Publish(event beat.Event, cursorUpdate interface{}) error { - panic("TODO: implement me") + if cursorUpdate == nil { + return c.forward(event) + } + + op, err := createUpdateOp(c.cursor.store, c.cursor.resource, cursorUpdate) + if err != nil { + return err + } + + event.Private = op + return c.forward(event) +} + +func (c *cursorPublisher) forward(event beat.Event) error { + c.client.Publish(event) + if c.canceler == nil { + return nil + } + return c.canceler.Err() +} + +func createUpdateOp(store *store, resource *resource, updates interface{}) (*updateOp, error) { + ts := time.Now() + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + cursor := resource.pendingCursor + if resource.activeCursorOperations == 0 { + var tmp interface{} + typeconv.Convert(&tmp, cursor) + resource.pendingCursor = tmp + cursor = tmp + } + if err := typeconv.Convert(&cursor, updates); err != nil { + return nil, err + } + resource.pendingCursor = cursor + + resource.Retain() + resource.activeCursorOperations++ + return &updateOp{ + resource: resource, + store: store, + timestamp: ts, + delta: updates, + }, nil +} + +// done releases resources held by the last N updateOps. +func (op *updateOp) done(n uint) { + op.resource.UpdatesReleaseN(n) + op.resource = nil + *op = updateOp{} } // Execute updates the persistent store with the scheduled changes and releases the resource. -func (op *updateOp) Execute(numEvents uint) { - panic("TODO: implement me") +func (op *updateOp) Execute(n uint) { + resource := op.resource + defer op.done(n) + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + resource.activeCursorOperations -= n + if resource.activeCursorOperations == 0 { + resource.cursor = resource.pendingCursor + resource.pendingCursor = nil + } else { + typeconv.Convert(&resource.cursor, op.delta) + } + + if resource.internalState.Updated.Before(op.timestamp) { + resource.internalState.Updated = op.timestamp + } + + err := op.store.persistentStore.Set(resource.key, resource.inSyncStateSnapshot()) + if err != nil { + if !statestore.IsClosed(err) { + op.store.log.Errorf("Failed to update state in the registry for '%v'", resource.key) + } + } else { + resource.internalInSync = true + resource.stored = true + } } diff --git a/filebeat/input/v2/input-cursor/publish_test.go b/filebeat/input/v2/input-cursor/publish_test.go new file mode 100644 index 00000000000..28c274baf94 --- /dev/null +++ b/filebeat/input/v2/input-cursor/publish_test.go @@ -0,0 +1,158 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" +) + +func TestPublish(t *testing.T) { + t.Run("event with cursor state creates update operation", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + var actual beat.Event + client := &pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { actual = event }, + } + publisher := cursorPublisher{nil, client, &cursor} + publisher.Publish(beat.Event{}, "test") + + require.NotNil(t, actual.Private) + }) + + t.Run("event without cursor creates no update operation", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + var actual beat.Event + client := &pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { actual = event }, + } + publisher := cursorPublisher{nil, client, &cursor} + publisher.Publish(beat.Event{}, nil) + require.Nil(t, actual.Private) + }) + + t.Run("publish returns error if context has been cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + cancel() + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + publisher := cursorPublisher{ctx, &pubtest.FakeClient{}, &cursor} + err := publisher.Publish(beat.Event{}, nil) + require.Equal(t, context.Canceled, err) + }) +} + +func TestOp_Execute(t *testing.T) { + t.Run("applying final op marks the key as finished", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create op and release resource. The 'resource' must still be active + op := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state") + res.Release() + require.False(t, res.Finished()) + + // this was the last op, the resource should become inactive + op.Execute(1) + require.True(t, res.Finished()) + + // validate state: + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + want := "test-updated-cursor-state" + assert.Equal(t, want, inSyncCursor) + assert.Equal(t, want, inMemCursor) + }) + + t.Run("acking multiple ops applies the latest update and marks key as finished", func(t *testing.T) { + // when acking N events, intermediate updates are dropped in favor of the latest update operation. + // This test checks that the resource is correctly marked as finished. + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create update operations and release resource. The 'resource' must still be active + mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-dropped") + op := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-final") + res.Release() + require.False(t, res.Finished()) + + // this was the last op, the resource should become inactive + op.Execute(2) + require.True(t, res.Finished()) + + // validate state: + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + want := "test-updated-cursor-state-final" + assert.Equal(t, want, inSyncCursor) + assert.Equal(t, want, inMemCursor) + }) + + t.Run("ACK only subset of pending ops will only update up to ACKed state", func(t *testing.T) { + // when acking N events, intermediate updates are dropped in favor of the latest update operation. + // This test checks that the resource is correctly marked as finished. + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create update operations and release resource. The 'resource' must still be active + op1 := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-intermediate") + op2 := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-final") + res.Release() + require.False(t, res.Finished()) + + defer op2.done(1) // cleanup after test + + // this was the intermediate op, the resource should still be active + op1.Execute(1) + require.False(t, res.Finished()) + + // validate state (in memory state is always up to data to most recent update): + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + assert.Equal(t, "test-updated-cursor-state-intermediate", inSyncCursor) + assert.Equal(t, "test-updated-cursor-state-final", inMemCursor) + }) +} + +func mustCreateUpdateOp(t *testing.T, store *store, resource *resource, updates interface{}) *updateOp { + op, err := createUpdateOp(store, resource, updates) + if err != nil { + t.Fatalf("Failed to create update op: %v", err) + } + return op +} diff --git a/filebeat/input/v2/input-cursor/store.go b/filebeat/input/v2/input-cursor/store.go index f44cf6760dc..66d5b450936 100644 --- a/filebeat/input/v2/input-cursor/store.go +++ b/filebeat/input/v2/input-cursor/store.go @@ -18,10 +18,12 @@ package cursor import ( + "strings" "sync" "time" "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/common/cleanup" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore" @@ -126,7 +128,25 @@ type ( var closeStore = (*store).close func openStore(log *logp.Logger, statestore StateStore, prefix string) (*store, error) { - panic("TODO: implement me") + ok := false + + persistentStore, err := statestore.Access() + if err != nil { + return nil, err + } + defer cleanup.IfNot(&ok, func() { persistentStore.Close() }) + + states, err := readStates(log, persistentStore, prefix) + if err != nil { + return nil, err + } + + ok = true + return &store{ + log: log, + persistentStore: persistentStore, + ephemeralStore: states, + }, nil } func (s *store) Retain() { s.refCount.Retain() } @@ -137,7 +157,9 @@ func (s *store) Release() { } func (s *store) close() { - panic("TODO: implement me") + if err := s.persistentStore.Close(); err != nil { + s.log.Errorf("Closing registry store did report an error: %+v", err) + } } // Get returns the resource for the key. @@ -152,18 +174,62 @@ func (s *store) Get(key string) *resource { // On update the resource its `cursor` state is used, to keep the cursor state in sync with the current known // on disk store state. func (s *store) UpdateTTL(resource *resource, ttl time.Duration) { - panic("TODO: implement me") + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + if resource.stored && resource.internalState.TTL == ttl { + return + } + + resource.internalState.TTL = ttl + if resource.internalState.Updated.IsZero() { + resource.internalState.Updated = time.Now() + } + + err := s.persistentStore.Set(resource.key, state{ + TTL: resource.internalState.TTL, + Updated: resource.internalState.Updated, + Cursor: resource.cursor, + }) + if err != nil { + s.log.Errorf("Failed to update resource management fields for '%v'", resource.key) + resource.internalInSync = false + } else { + resource.stored = true + resource.internalInSync = true + } } // Find returns the resource for a given key. If the key is unknown and create is set to false nil will be returned. // The resource returned by Find is marked as active. (*resource).Release must be called to mark the resource as inactive again. func (s *states) Find(key string, create bool) *resource { - panic("TODO: implement me") + s.mu.Lock() + defer s.mu.Unlock() + + if resource := s.table[key]; resource != nil { + resource.Retain() + return resource + } + + if !create { + return nil + } + + // resource is owned by table(session) and input that uses the resource. + resource := &resource{ + stored: false, + key: key, + lock: unison.MakeMutex(), + } + s.table[key] = resource + resource.Retain() + return resource } // IsNew returns true if we have no state recorded for the current resource. func (r *resource) IsNew() bool { - panic("TODO: implement me") + r.stateMutex.Lock() + defer r.stateMutex.Unlock() + return r.pendingCursor == nil && r.cursor == nil } // Retain is used to indicate that 'resource' gets an additional 'owner'. @@ -201,3 +267,58 @@ func (r *resource) inSyncStateSnapshot() state { Cursor: r.cursor, } } + +// stateSnapshot returns the current in memory state, that already contains state updates +// not yet ACKed. +func (r *resource) stateSnapshot() state { + cursor := r.pendingCursor + if r.activeCursorOperations == 0 { + cursor = r.cursor + } + + return state{ + TTL: r.internalState.TTL, + Updated: r.internalState.Updated, + Cursor: cursor, + } +} + +func readStates(log *logp.Logger, store *statestore.Store, prefix string) (*states, error) { + keyPrefix := prefix + "::" + states := &states{ + table: map[string]*resource{}, + } + + err := store.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + if !strings.HasPrefix(string(key), keyPrefix) { + return true, nil + } + + var st state + if err := dec.Decode(&st); err != nil { + log.Errorf("Failed to read regisry state for '%v', cursor state will be ignored. Error was: %+v", + key, err) + return true, nil + } + + resource := &resource{ + key: key, + stored: true, + lock: unison.MakeMutex(), + internalInSync: true, + internalState: stateInternal{ + TTL: st.TTL, + Updated: st.Updated, + }, + cursor: st.Cursor, + } + states.table[resource.key] = resource + + return true, nil + }) + + if err != nil { + return nil, err + } + return states, nil +} diff --git a/filebeat/input/v2/input-cursor/store_test.go b/filebeat/input/v2/input-cursor/store_test.go new file mode 100644 index 00000000000..2fa3f6eaa3e --- /dev/null +++ b/filebeat/input/v2/input-cursor/store_test.go @@ -0,0 +1,351 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/storetest" +) + +type testStateStore struct { + Store *statestore.Store + GCPeriod time.Duration +} + +func TestStore_OpenClose(t *testing.T) { + t.Run("releasing store closes", func(t *testing.T) { + var closed bool + cleanup := closeStoreWith(func(s *store) { + closed = true + s.close() + }) + defer cleanup() + + store := testOpenStore(t, "test", nil) + store.Release() + + require.True(t, closed) + }) + + t.Run("fail if persistent store can not be accessed", func(t *testing.T) { + _, err := openStore(logp.NewLogger("test"), testStateStore{}, "test") + require.Error(t, err) + }) + + t.Run("load from empty", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + require.Equal(t, 0, len(storeMemorySnapshot(store))) + require.Equal(t, 0, len(storeInSyncSnapshot(store))) + }) + + t.Run("already available state is loaded", func(t *testing.T) { + states := map[string]state{ + "test::key0": {Cursor: "1"}, + "test::key1": {Cursor: "2"}, + } + + store := testOpenStore(t, "test", createSampleStore(t, states)) + defer store.Release() + + checkEqualStoreState(t, states, storeMemorySnapshot(store)) + checkEqualStoreState(t, states, storeInSyncSnapshot(store)) + }) + + t.Run("ignore entries with wrong index on open", func(t *testing.T) { + states := map[string]state{ + "test::key0": {Cursor: "1"}, + "other::key": {Cursor: "2"}, + } + + store := testOpenStore(t, "test", createSampleStore(t, states)) + defer store.Release() + + want := map[string]state{ + "test::key0": {Cursor: "1"}, + } + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) +} + +func TestStore_Get(t *testing.T) { + t.Run("find existing resource", func(t *testing.T) { + cursorState := state{Cursor: "1"} + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key0": cursorState, + })) + defer store.Release() + + res := store.Get("test::key0") + require.NotNil(t, res) + defer res.Release() + + // check in memory state matches matches original persistent state + require.Equal(t, cursorState, res.stateSnapshot()) + // check assumed in-sync state matches matches original persistent state + require.Equal(t, cursorState, res.inSyncStateSnapshot()) + }) + + t.Run("access unknown resource", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + require.NotNil(t, res) + defer res.Release() + + // new resource has empty state + require.Equal(t, state{}, res.stateSnapshot()) + }) + + t.Run("same resource is returned", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res1 := store.Get("test::key") + require.NotNil(t, res1) + defer res1.Release() + + res2 := store.Get("test::key") + require.NotNil(t, res2) + defer res2.Release() + + assert.Equal(t, res1, res2) + }) +} + +func TestStore_UpdateTTL(t *testing.T) { + t.Run("add TTL for new entry to store", func(t *testing.T) { + // when creating a resource we set the TTL and insert a new key value pair without cursor value into the store: + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + store.UpdateTTL(res, 60*time.Second) + + want := map[string]state{ + "test::key": { + TTL: 60 * time.Second, + Updated: res.internalState.Updated, + Cursor: nil, + }, + } + + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) + + t.Run("update TTL for in-sync resource does not overwrite state", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Second, + Cursor: "test", + }, + })) + defer store.Release() + + res := store.Get("test::key") + store.UpdateTTL(res, 60*time.Second) + want := map[string]state{ + "test::key": { + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test", + }, + } + + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) + + t.Run("update TTL for resource with pending updates", func(t *testing.T) { + // This test updates the resource TTL while update operations are still + // pending, but not synced to the persistent store yet. + // UpdateTTL changes the state in the persistent store immediately, and must therefore + // serialize the old in-sync state with update meta-data. + + // create store + backend := createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Second, + Cursor: "test", + }, + }) + store := testOpenStore(t, "test", backend) + defer store.Release() + + // create pending update operation + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + // Update key/value pair TTL. This will update the internal state in the + // persistent store only, not modifying the old cursor state yet. + store.UpdateTTL(res, 60*time.Second) + + // validate + wantMemoryState := state{ + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test-state-update", + } + wantInSyncState := state{ + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test", + } + + checkEqualStoreState(t, map[string]state{"test::key": wantMemoryState}, storeMemorySnapshot(store)) + checkEqualStoreState(t, map[string]state{"test::key": wantInSyncState}, storeInSyncSnapshot(store)) + checkEqualStoreState(t, map[string]state{"test::key": wantInSyncState}, backend.snapshot()) + }) +} + +func closeStoreWith(fn func(s *store)) func() { + old := closeStore + closeStore = fn + return func() { + closeStore = old + } +} + +func testOpenStore(t *testing.T, prefix string, persistentStore StateStore) *store { + if persistentStore == nil { + persistentStore = createSampleStore(t, nil) + } + + store, err := openStore(logp.NewLogger("test"), persistentStore, prefix) + if err != nil { + t.Fatalf("failed to open the store") + } + return store +} + +func createSampleStore(t *testing.T, data map[string]state) testStateStore { + storeReg := statestore.NewRegistry(storetest.NewMemoryStoreBackend()) + store, err := storeReg.Get("test") + if err != nil { + t.Fatalf("Failed to access store: %v", err) + } + + for k, v := range data { + if err := store.Set(k, v); err != nil { + t.Fatalf("Error when populating the sample store: %v", err) + } + } + + return testStateStore{ + Store: store, + } +} + +func (ts testStateStore) WithGCPeriod(d time.Duration) testStateStore { ts.GCPeriod = d; return ts } +func (ts testStateStore) CleanupInterval() time.Duration { return ts.GCPeriod } +func (ts testStateStore) Access() (*statestore.Store, error) { + if ts.Store == nil { + return nil, errors.New("no store configured") + } + return ts.Store, nil +} + +// snapshot copies all key/value pairs from the persistent store into a table for inspection. +func (ts testStateStore) snapshot() map[string]state { + states := map[string]state{} + err := ts.Store.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + var st state + if err := dec.Decode(&st); err != nil { + return false, err + } + states[key] = st + return true, nil + }) + + if err != nil { + panic("unexpected decode error from persistent test store") + } + return states +} + +// storeMemorySnapshot copies all key/value pairs into a table for inspection. +// The state returned reflects the in memory state, which can be ahead of the +// persistent state. +// +// Note: The state returned by storeMemorySnapshot is always ahead of the state returned by storeInSyncSnapshot. +// All key value pairs are fully in-sync, if both snapshot functions return the same state. +func storeMemorySnapshot(store *store) map[string]state { + store.ephemeralStore.mu.Lock() + defer store.ephemeralStore.mu.Unlock() + + states := map[string]state{} + for k, res := range store.ephemeralStore.table { + states[k] = res.stateSnapshot() + } + return states +} + +// storeInSyncSnapshot copies all key/value pairs into the table for inspection. +// The state returned reflects the current state that the in-memory tables assumed to be +// written to the persistent store already. + +// Note: The state returned by storeMemorySnapshot is always ahead of the state returned by storeInSyncSnapshot. +// All key value pairs are fully in-sync, if both snapshot functions return the same state. +func storeInSyncSnapshot(store *store) map[string]state { + store.ephemeralStore.mu.Lock() + defer store.ephemeralStore.mu.Unlock() + + states := map[string]state{} + for k, res := range store.ephemeralStore.table { + states[k] = res.inSyncStateSnapshot() + } + return states +} + +// checkEqualStoreState compares 2 store snapshot tables for equality. The test +// fails with Errorf if the state differ. +// +// Note: testify is too strict when comparing timestamp, better use checkEqualStoreState. +func checkEqualStoreState(t *testing.T, want, got map[string]state) bool { + if d := cmp.Diff(want, got); d != "" { + t.Errorf("store state mismatch (-want +got):\n%s", d) + return false + } + return true +} + +// requireEqualStoreState compares 2 store snapshot tables for equality. The test +// fails with Fatalf if the state differ. +// +// Note: testify is too strict when comparing timestamp, better use checkEqualStoreState. +func requireEqualStoreState(t *testing.T, want, got map[string]state) bool { + if d := cmp.Diff(want, got); d != "" { + t.Fatalf("store state mismatch (-want +got):\n%s", d) + return false + } + return true +} diff --git a/filebeat/module/kafka/_meta/docs.asciidoc b/filebeat/module/kafka/_meta/docs.asciidoc index 4e199f98b4b..4940b08d54e 100644 --- a/filebeat/module/kafka/_meta/docs.asciidoc +++ b/filebeat/module/kafka/_meta/docs.asciidoc @@ -6,6 +6,8 @@ The +{modulename}+ module collects and parses the logs created by https://kafka.apache.org/[Kafka]. +The module has additional support for parsing thread ID from logs. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/filebeat/module/kafka/fields.go b/filebeat/module/kafka/fields.go index 86950968bf0..6d0aadfd867 100644 --- a/filebeat/module/kafka/fields.go +++ b/filebeat/module/kafka/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKafka returns asset data. // This is the base64 encoded gzipped contents of module/kafka. func AssetKafka() string { - return "eJykkjtuwzAQRHudYuDeOoCKNOkSpMsFFtZKJsQfyLUT3z6gGP8Y2paR7cQRZ55Gu8bEhw4TDRM1gCjR3GH1np5XDdBz3ATlRTnb4aUBgFmDcf1OcwMMinUfu1law5Lhs10aOXjuMAa3878nFc9rm0sr7cbTWc3spmGeDKvdCK0sx/ZCLBOvUnnP+ko5ZpNWFAvFk2xn0rZ2z6gxUGaTsONqnuEYaeQnE+u3buVVgzfOeGfZSjV64sOXC32h3ak7zevRErLluXoVU46yI4bgTFsH0RTLr/wHxBvtKXs+RSGBNvW/UO7cAgbgM9lB2RNDWsK2eLG2h4+aedTOIro/LUnGvdMTHu7smUz4u9yqhVjAR/aGpyBww5mubX4CAAD//40BKYM=" + return "eJykkztywyAQhnud4h/31gFUpEmXTDpfYMdayYwQMLB24ttngPhF8GtMZ9Z8/6eFXWLifYeJhokaQJRo7rD4jL8XDdBzWHvlRFnT4a0BgFTDbPut5gYYFOs+dKm0hKGZT7i4ZO+4w+jt1v3tVJiXmHOUtuNxrwa7Cswry2o7QivDoT0rlokXqbxjfVE5ZJNWFIqKI9kk07Z2blajp+wmfsvVvJlDoJGfTKyfupZXDV7b2VnDRqrRE++/re+L2o12x/V+QEI2nFqvQsxRZsTg7dzWRTSF8itfkPigHWXmUxay8Uxl1Asaq8RL8Oc8PK3rr6F8+w9IAKuIgzJHhzgMbfHH2jzgzg3da89Ddv9uS7LujT7h7uyczIR/ytf9oBbwldlw5AV2ONm1zW8AAAD//95/TD8=" } diff --git a/filebeat/module/kafka/log/_meta/fields.yml b/filebeat/module/kafka/log/_meta/fields.yml index c06f44d0bf4..32b9008f748 100644 --- a/filebeat/module/kafka/log/_meta/fields.yml +++ b/filebeat/module/kafka/log/_meta/fields.yml @@ -20,6 +20,10 @@ type: keyword description: > Java class the log is coming from. + - name: thread + type: keyword + description: > + Thread name the log is coming from. - name: trace type: group description: > diff --git a/filebeat/module/kafka/log/ingest/pipeline.yml b/filebeat/module/kafka/log/ingest/pipeline.yml index 41db8f4f197..a1072489122 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.yml +++ b/filebeat/module/kafka/log/ingest/pipeline.yml @@ -6,6 +6,7 @@ processors: patterns: - (?m)%{TIMESTAMP_ISO8601:kafka.log.timestamp}. %{LOGLEVEL:log.level} +%{JAVALOGMESSAGE:message} \(%{JAVACLASS:kafka.log.class}\)$[ \n]*(?'kafka.log.trace.full'.*) + - (?m)\[%{TIMESTAMP_ISO8601:kafka.log.timestamp}\] \[%{LOGLEVEL:log.level} ?\] \[%{NOTSPACE:kafka.log.thread}\] \[%{NOTSPACE:kafka.log.class}\] \- %{GREEDYDATA:message} - grok: field: message pattern_definitions: diff --git a/filebeat/module/kafka/log/test/state-change-2.2.2.log b/filebeat/module/kafka/log/test/state-change-2.2.2.log new file mode 100644 index 00000000000..69d64c90d22 --- /dev/null +++ b/filebeat/module/kafka/log/test/state-change-2.2.2.log @@ -0,0 +1,23 @@ +[2020-04-23 00:28:21,796] [WARN ] [kafka-request-handler-7] [state.change.logger] - [Broker id=1] Ignoring LeaderAndIsr request from controller 1 with correlation id 1 epoch 27 for partition xxx.xxx.inventory.test.1-0-2 since its associated leader epoch 5 is not higher than the current leader epoch 5 +[2020-01-20 01:32:00,705] [ERROR] [controller-event-thread] [state.change.logger] - [Controller id=1 epoch=25] Controller 1 epoch 25 failed to change state for partition xxx.xxx.na.och.aud.1-0-2 from OfflinePartition to OnlinePartition +kafka.common.StateChangeFailedException: Failed to elect leader for partition xxx.xxx.na.och.aud.1-0-2 under strategy OfflinePartitionLeaderElectionStrategy + at kafka.controller.PartitionStateMachine.$anonfun$doElectLeaderForPartitions$9(PartitionStateMachine.scala:390) + at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) + at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) + at kafka.controller.PartitionStateMachine.doElectLeaderForPartitions(PartitionStateMachine.scala:388) + at kafka.controller.PartitionStateMachine.electLeaderForPartitions(PartitionStateMachine.scala:315) + at kafka.controller.PartitionStateMachine.doHandleStateChanges(PartitionStateMachine.scala:225) + at kafka.controller.PartitionStateMachine.handleStateChanges(PartitionStateMachine.scala:141) + at kafka.controller.PartitionStateMachine.triggerOnlinePartitionStateChange(PartitionStateMachine.scala:123) + at kafka.controller.PartitionStateMachine.triggerOnlinePartitionStateChange(PartitionStateMachine.scala:109) + at kafka.controller.PartitionStateMachine.startup(PartitionStateMachine.scala:66) + at kafka.controller.KafkaController.onControllerFailover(KafkaController.scala:266) + at kafka.controller.KafkaController.kafka$controller$KafkaController$$elect(KafkaController.scala:1271) + at kafka.controller.KafkaController$Startup$.process(KafkaController.scala:1184) + at kafka.controller.ControllerEventManager$ControllerEventThread.$anonfun$doWork$1(ControllerEventManager.scala:94) + at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) + at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) + at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:94) + at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) + diff --git a/filebeat/module/kafka/log/test/state-change-2.2.2.log-expected.json b/filebeat/module/kafka/log/test/state-change-2.2.2.log-expected.json new file mode 100644 index 00000000000..a8ad3645ec7 --- /dev/null +++ b/filebeat/module/kafka/log/test/state-change-2.2.2.log-expected.json @@ -0,0 +1,39 @@ +[ + { + "@timestamp": "2020-04-23T00:28:21.796-02:00", + "event.dataset": "kafka.log", + "event.kind": "event", + "event.module": "kafka", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "kafka.log.class": "state.change.logger", + "kafka.log.component": "Broker id=1", + "kafka.log.thread": "kafka-request-handler-7", + "log.level": "WARN", + "log.offset": 0, + "message": "Ignoring LeaderAndIsr request from controller 1 with correlation id 1 epoch 27 for partition xxx.xxx.inventory.test.1-0-2 since its associated leader epoch 5 is not higher than the current leader epoch 5 ", + "service.type": "kafka" + }, + { + "@timestamp": "2020-01-20T01:32:00.705-02:00", + "event.dataset": "kafka.log", + "event.kind": "event", + "event.module": "kafka", + "event.timezone": "-02:00", + "event.type": "error", + "fileset.name": "log", + "input.type": "log", + "kafka.log.class": "state.change.logger", + "kafka.log.component": "Controller id=1 epoch=25", + "kafka.log.thread": "controller-event-thread", + "log.flags": [ + "multiline" + ], + "log.level": "ERROR", + "log.offset": 303, + "message": "Controller 1 epoch 25 failed to change state for partition xxx.xxx.na.och.aud.1-0-2 from OfflinePartition to OnlinePartition", + "service.type": "kafka" + } +] \ No newline at end of file diff --git a/go.mod b/go.mod index f4867e4d91e..44231c67d65 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 github.com/elastic/ecs v1.5.0 github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07 - github.com/elastic/go-concert v0.0.2 + github.com/elastic/go-concert v0.0.3 github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 github.com/elastic/go-licenser v0.3.1 github.com/elastic/go-lookslike v0.3.0 @@ -139,10 +139,11 @@ require ( github.com/shirou/gopsutil v2.19.11+incompatible github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.5 + github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.6.1 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786 - github.com/urso/ecslog v0.0.1 + github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 // indirect @@ -163,7 +164,7 @@ require ( golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 - golang.org/x/tools v0.0.0-20200630154851-b2d8b0336632 + golang.org/x/tools v0.0.0-20200701041122-1837592efa10 google.golang.org/api v0.15.0 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb google.golang.org/grpc v1.29.1 @@ -179,7 +180,7 @@ require ( replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible - github.com/Shopify/sarama => github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e + github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec github.com/cucumber/godog => github.com/cucumber/godog v0.8.1 github.com/docker/docker => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f diff --git a/go.sum b/go.sum index 31b5a70a086..ef0fde2efdb 100644 --- a/go.sum +++ b/go.sum @@ -98,7 +98,6 @@ github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc h github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -107,7 +106,6 @@ github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 h1:7rj9qZ63knnVo2Z github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20/go.mod h1:cI59GRkC2FRaFYtgbYEqMlgnnfvAwXzjojyZKXwklNg= github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 h1:WFwa9pqou0Nb4DdfBOyaBTH0GqLE74Qwdf61E7ITHwQ= github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43/go.mod h1:tJPYQG4mnMeUtQvQKNkbsFrnmZOg59Qnf8CcctFv5v4= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5 h1:nkZ9axP+MvUFCu8JRN/MCY+DmTfs6lY7hE0QnJbxSdI= github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= @@ -175,7 +173,6 @@ github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQa github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cucumber/godog v0.8.1 h1:lVb+X41I4YDreE+ibZ50bdXmySxgRviYFgKY6Aw4XE8= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= @@ -231,8 +228,8 @@ github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07 h1 github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng= -github.com/elastic/go-concert v0.0.2 h1:hJb9h99LS/lyjf7pE1wQ+eiNw+0CXVLCJR42yx+AvOQ= -github.com/elastic/go-concert v0.0.2/go.mod h1:9MtFarjXroUgmm0m6HY3NSe1XiKhdktiNRRj9hWvIaM= +github.com/elastic/go-concert v0.0.3 h1:f0F4WOi8tBOFIgwA7YbHRQ+Ok8vR+/qFrG7vYvbpX5Q= +github.com/elastic/go-concert v0.0.3/go.mod h1:9MtFarjXroUgmm0m6HY3NSe1XiKhdktiNRRj9hWvIaM= github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 h1:Jcnojiuok7Ea5hitJK9VWmBigganE2MMETOH0VZasEA= github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8/go.mod h1:j2CZcVcluWDGhQTnq1SOPy1NKEIa74FtQ39Nnz87Jxk= github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrEzObU= @@ -247,8 +244,6 @@ github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f h1:Fvsq github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f/go.mod h1:OPGqFNdTS34kMReS5hPFtBhD9J8itmSDurs1ix2wx7c= github.com/elastic/go-seccomp-bpf v1.1.0 h1:jUzzDc6LyCtdolZdvL/26dad6rZ9vsc7xZ2eadKECAU= github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8= -github.com/elastic/go-structform v0.0.6 h1:wqeK4LwD2NNDOoRGTImE24S6pkCDVr8+oUSIkmChzLk= -github.com/elastic/go-structform v0.0.6/go.mod h1:QrMyP3oM9Sjk92EVGLgRaL2lKt0Qx7ZNDRWDxB6khVs= github.com/elastic/go-structform v0.0.7 h1:ihszOJQryNuIIHE2ZgsbiDq+agKO6V4yK0JYAI3tjzc= github.com/elastic/go-structform v0.0.7/go.mod h1:QrMyP3oM9Sjk92EVGLgRaL2lKt0Qx7ZNDRWDxB6khVs= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= @@ -264,12 +259,11 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gosigar v0.10.5 h1:GzPQ+78RaAb4J63unidA/JavQRKrB6s8IOzN6Ib59jo= github.com/elastic/gosigar v0.10.5/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= -github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e h1:2jm3380rkaGcosRpvtgIQrl7F5Cb99aFJYis7Y5hoJw= -github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE= +github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec h1:rAHd7DeHIHjSzvnkl197GKh9TCWGKg/z2BBbbGOEiWI= +github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -278,7 +272,6 @@ github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.5.0 h1:vBh+kQp8lg9XPr56u1CPrWjFXtdphMoGWVHr9/1c+A0= github.com/fatih/color v1.5.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= @@ -287,7 +280,6 @@ github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24 h1:nREVDi4H8mwnN github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= @@ -441,7 +433,6 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -472,7 +463,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/justinas/nosurf v1.1.0/go.mod h1:ALpWdSbuNGy2lZWtyXdjkYv4edL23oSEgfBT1gPJ5BQ= github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA= github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -488,7 +478,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -523,7 +512,6 @@ github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v1.0.1 h1:x0jD3dcHk9a9xPSDN6YEL4xL6Qz0dvNYm8yZqui5chI= github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4= github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 h1:qdHlMllk/PTLUrX3XdtXDrLL1lPSfcqUmJD1eYfbapg= @@ -572,7 +560,6 @@ github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -652,7 +639,6 @@ github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzu github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -673,18 +659,12 @@ github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786 h1:B/IVHYiI0d04dudYw+CvCAGqSMq8d0yWy56eD6p85BQ= github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786/go.mod h1:RIkfovP3Y7my19aXEjjbNd9E5TlHozzAyt7B8AaEcwg= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urso/diag v0.0.0-20200125202105-ffdc32ff5518/go.mod h1:pNWFTeQ+V1OYT/TzWpnWb6eQBdoXpdx+H+lrH97/Oyo= github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 h1:OHNw/6pXODJAB32NujjdQO/KIYQ3KAbHQfCzH81XdCs= github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797/go.mod h1:pNWFTeQ+V1OYT/TzWpnWb6eQBdoXpdx+H+lrH97/Oyo= -github.com/urso/diag-ecs v0.0.0-20200210114345-ab085841dcb9 h1:GzsakegdLNhw0mF2fcFd+BgdY8owV+4Y+6MvbgRXIWg= -github.com/urso/diag-ecs v0.0.0-20200210114345-ab085841dcb9/go.mod h1:+1Ug5A104KCMD8ZZ4YarKGNSt8ANJWA7nWqji37BmrQ= -github.com/urso/ecslog v0.0.1 h1:Candk+au3CbAT5SpVMhTE3VIq0r09UJely7vUzVAfF0= -github.com/urso/ecslog v0.0.1/go.mod h1:wky/kxUyw0VJw/HlXal+7oTT2YDU1KAWrxbuKRFblEI= github.com/urso/go-bin v0.0.0-20180220135811-781c575c9f0e h1:NiofbjIUI5gR+ybDsGSVH1fWyjSeDYiYVJHT1+kcsak= github.com/urso/go-bin v0.0.0-20180220135811-781c575c9f0e/go.mod h1:6GfHrdWBQYjFRIznu7XuQH4lYB2w8nO4bnImVKkzPOM= +github.com/urso/magetools v0.0.0-20190919040553-290c89e0c230 h1:Ft1EJ6JL0F/RV6o2qJ1Be+wYxjYUSfRA3srfHgSgojc= github.com/urso/magetools v0.0.0-20190919040553-290c89e0c230/go.mod h1:DFxTNgS/ExCGmmjVjSOgS2WjtfjKXgCyDzAFgbtovSA= -github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a h1:jWAaRFnay3H2e6S0GGCl5nKrkgQNlarCE/kvcutzBmw= -github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a/go.mod h1:DbaJnRzkGaWrMWm5Hz6QVnUj//x9/zjrfx8bF3J+GJY= github.com/urso/qcgen v0.0.0-20180131103024-0b059e7db4f4 h1:hhA8EBThzz9PztawVTycKvfETVuBqxAQ5keFlAVtbAw= github.com/urso/qcgen v0.0.0-20180131103024-0b059e7db4f4/go.mod h1:RspW+E2Yb7Fs7HclB2tiDaiu6Rp41BiIG4Wo1YaoXGc= github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec h1:HkZIDJrMKZHPsYhmH2XjTTSk1pbMCFfpxSnyzZUFm+k= @@ -694,7 +674,6 @@ github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 h1:NeNpIvfvaFOh0BH7 github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/xanzy/go-gitlab v0.22.3 h1:/rNlZ2hquUWNc6rJdntVM03tEOoTmnZ1lcNyJCl0WlU= github.com/xanzy/go-gitlab v0.22.3/go.mod h1:t4Bmvnxj7k37S4Y17lfLx+nLqkf/oQwT2HagfWKv5Og= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= @@ -740,12 +719,10 @@ go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -830,7 +807,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -840,7 +816,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -937,12 +912,8 @@ gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 h1:/saqWwm73dLmuzbNhe92F0QsZ/KiFND+esHco2v1hiY= gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/journalbeat/config/config.go b/journalbeat/config/config.go index a5d9127641d..1771a2a4e7c 100644 --- a/journalbeat/config/config.go +++ b/journalbeat/config/config.go @@ -21,56 +21,18 @@ package config import ( - "fmt" - "github.com/elastic/beats/v7/libbeat/common" ) -// SeekMode is specifies how a journal is read -type SeekMode uint8 - // Config stores the configuration of Journalbeat type Config struct { Inputs []*common.Config `config:"inputs"` RegistryFile string `config:"registry_file"` } -const ( - // SeekInvalid is an invalid value for seek - SeekInvalid SeekMode = iota - // SeekHead option seeks to the head of a journal - SeekHead - // SeekTail option seeks to the tail of a journal - SeekTail - // SeekCursor option seeks to the position specified in the cursor - SeekCursor - - seekHeadStr = "head" - seekTailStr = "tail" - seekCursorStr = "cursor" -) - var ( // DefaultConfig are the defaults of a Journalbeat instance DefaultConfig = Config{ RegistryFile: "registry", } - - seekModes = map[string]SeekMode{ - seekHeadStr: SeekHead, - seekTailStr: SeekTail, - seekCursorStr: SeekCursor, - } ) - -// Unpack validates and unpack "seek" config option -func (m *SeekMode) Unpack(value string) error { - mode, ok := seekModes[value] - if !ok { - return fmt.Errorf("invalid seek mode '%s'", value) - } - - *m = mode - - return nil -} diff --git a/journalbeat/config/config_test.go b/journalbeat/config/config_test.go deleted file mode 100644 index f842b812b1f..00000000000 --- a/journalbeat/config/config_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package config - -import ( - "testing" -) - -func TestUnpack(t *testing.T) { - - tests := []struct { - mode SeekMode - modeStr string - }{ - { - mode: SeekHead, - modeStr: seekHeadStr, - }, - { - mode: SeekTail, - modeStr: seekTailStr, - }, - { - mode: SeekCursor, - modeStr: seekCursorStr, - }, - } - - for _, tc := range tests { - tc := tc - - t.Run(tc.modeStr, func(t *testing.T) { - t.Parallel() - - m := SeekInvalid - err := m.Unpack(tc.modeStr) - if err != nil { - t.Fatal(err) - } - - if m != tc.mode { - t.Errorf("wrong mode, expected %v, got %v", tc.mode, m) - } - }) - } -} - -func TestUnpackFailure(t *testing.T) { - - tests := []struct { - modeStr string - }{ - { - modeStr: "invalid", - }, - { - modeStr: "", - }, - { - modeStr: "unknown", - }, - } - - for _, tc := range tests { - tc := tc - - t.Run(tc.modeStr, func(t *testing.T) { - t.Parallel() - - m := SeekInvalid - err := m.Unpack(tc.modeStr) - if err == nil { - t.Errorf("an error was expected, got %v", m) - } - }) - } -} diff --git a/journalbeat/input/config.go b/journalbeat/input/config.go index 63c31ccfce7..6c202030b35 100644 --- a/journalbeat/input/config.go +++ b/journalbeat/input/config.go @@ -20,7 +20,8 @@ package input import ( "time" - "github.com/elastic/beats/v7/journalbeat/config" + "github.com/elastic/beats/v7/journalbeat/pkg/journalfield" + "github.com/elastic/beats/v7/journalbeat/pkg/journalread" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/libbeat/processors" @@ -38,11 +39,11 @@ type Config struct { // MaxBackoff is the limit of the backoff time. MaxBackoff time.Duration `config:"max_backoff" validate:"min=0,nonzero"` // Seek is the method to read from journals. - Seek config.SeekMode `config:"seek"` + Seek journalread.SeekMode `config:"seek"` // CursorSeekFallback sets where to seek if registry file is not available. - CursorSeekFallback config.SeekMode `config:"cursor_seek_fallback"` + CursorSeekFallback journalread.SeekMode `config:"cursor_seek_fallback"` // Matches store the key value pairs to match entries. - Matches []string `config:"include_matches"` + Matches []journalfield.Matcher `config:"include_matches"` // SaveRemoteHostname defines if the original source of the entry needs to be saved. SaveRemoteHostname bool `config:"save_remote_hostname"` @@ -59,8 +60,8 @@ var ( DefaultConfig = Config{ Backoff: 1 * time.Second, MaxBackoff: 20 * time.Second, - Seek: config.SeekCursor, - CursorSeekFallback: config.SeekHead, + Seek: journalread.SeekCursor, + CursorSeekFallback: journalread.SeekHead, SaveRemoteHostname: false, } ) diff --git a/journalbeat/pkg/journalfield/conv.go b/journalbeat/pkg/journalfield/conv.go new file mode 100644 index 00000000000..47214c3f91d --- /dev/null +++ b/journalbeat/pkg/journalfield/conv.go @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalfield + +import ( + "fmt" + "strconv" + "strings" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +// FieldConversion provides the mappings and conversion rules for raw fields of journald entries. +type FieldConversion map[string]Conversion + +// Conversion configures the conversion rules for a field. +type Conversion struct { + Names []string + IsInteger bool + Dropped bool +} + +// Converter applis configured conversion rules to journald entries, producing +// a new common.MapStr. +type Converter struct { + log *logp.Logger + conversions FieldConversion +} + +// NewConverter creates a new Converter from the given conversion rules. If +// conversions is nil, internal default conversion rules will be applied. +func NewConverter(log *logp.Logger, conversions FieldConversion) *Converter { + if conversions == nil { + conversions = journaldEventFields + } + + return &Converter{log: log, conversions: conversions} +} + +// Convert creates a common.MapStr from the raw fields by applying the +// configured conversion rules. +// Field type conversion errors are logged to at debug level and the original +// value is added to the map. +func (c *Converter) Convert(entryFields map[string]string) common.MapStr { + fields := common.MapStr{} + var custom common.MapStr + + for entryKey, v := range entryFields { + if fieldConversionInfo, ok := c.conversions[entryKey]; !ok { + if custom == nil { + custom = common.MapStr{} + } + normalized := strings.ToLower(strings.TrimLeft(entryKey, "_")) + custom.Put(normalized, v) + } else if !fieldConversionInfo.Dropped { + value, err := convertValue(fieldConversionInfo, v) + if err != nil { + value = v + c.log.Debugf("Journald mapping error: %v", err) + } + for _, name := range fieldConversionInfo.Names { + fields.Put(name, value) + } + } + } + + if len(custom) != 0 { + fields.Put("journald.custom", custom) + } + + return fields +} + +func convertValue(fc Conversion, value string) (interface{}, error) { + if fc.IsInteger { + v, err := strconv.ParseInt(value, 10, 64) + if err != nil { + // On some versions of systemd the 'syslog.pid' can contain the username + // appended to the end of the pid. In most cases this does not occur + // but in the cases that it does, this tries to strip ',\w*' from the + // value and then perform the conversion. + s := strings.Split(value, ",") + v, err = strconv.ParseInt(s[0], 10, 64) + if err != nil { + return value, fmt.Errorf("failed to convert field %s \"%v\" to int: %v", fc.Names[0], value, err) + } + } + return v, nil + } + return value, nil +} + +// helpers for creating a field conversion table. + +var ignoredField = Conversion{Dropped: true} + +func text(names ...string) Conversion { + return Conversion{Names: names, IsInteger: false, Dropped: false} +} + +func integer(names ...string) Conversion { + return Conversion{Names: names, IsInteger: true, Dropped: false} +} diff --git a/journalbeat/pkg/journalfield/conv_test.go b/journalbeat/pkg/journalfield/conv_test.go new file mode 100644 index 00000000000..a6514a95545 --- /dev/null +++ b/journalbeat/pkg/journalfield/conv_test.go @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build linux,cgo + +package journalfield + +import ( + "testing" + + "github.com/coreos/go-systemd/v22/sdjournal" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestConversion(t *testing.T) { + tests := map[string]struct { + fields map[string]string + want common.MapStr + }{ + "field name from fields.go": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_BOOT_ID: "123456", + }, + want: common.MapStr{ + "host": common.MapStr{ + "boot_id": "123456", + }, + }, + }, + "'syslog.pid' field without user append": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "pid": int64(123456), + }, + }, + }, + "'syslog.priority' field with junk": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_PRIORITY: "123456, ", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "priority": int64(123456), + }, + "log": common.MapStr{ + "syslog": common.MapStr{ + "priority": int64(123456), + }, + }, + }, + }, + "'syslog.pid' field with user append": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456,root", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "pid": int64(123456), + }, + }, + }, + "'syslog.pid' field empty": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "pid": "", + }, + }, + }, + "custom field": { + fields: map[string]string{ + "my_custom_field": "value", + }, + want: common.MapStr{ + "journald": common.MapStr{ + "custom": common.MapStr{ + "my_custom_field": "value", + }, + }, + }, + }, + "dropped field": { + fields: map[string]string{ + "_SOURCE_MONOTONIC_TIMESTAMP": "value", + }, + want: common.MapStr{}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + log := logp.NewLogger("test") + converted := NewConverter(log, nil).Convert(test.fields) + assert.Equal(t, test.want, converted) + }) + } +} diff --git a/journalbeat/pkg/journalfield/default.go b/journalbeat/pkg/journalfield/default.go new file mode 100644 index 00000000000..a8b3860e956 --- /dev/null +++ b/journalbeat/pkg/journalfield/default.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build linux,cgo + +package journalfield + +import "github.com/coreos/go-systemd/v22/sdjournal" + +// journaldEventFields provides default field mappings and conversions rules. +var journaldEventFields = FieldConversion{ + // provided by systemd journal + "COREDUMP_UNIT": text("journald.coredump.unit"), + "COREDUMP_USER_UNIT": text("journald.coredump.user_unit"), + "OBJECT_AUDIT_LOGINUID": integer("journald.object.audit.login_uid"), + "OBJECT_AUDIT_SESSION": integer("journald.object.audit.session"), + "OBJECT_CMDLINE": text("journald.object.cmd"), + "OBJECT_COMM": text("journald.object.name"), + "OBJECT_EXE": text("journald.object.executable"), + "OBJECT_GID": integer("journald.object.gid"), + "OBJECT_PID": integer("journald.object.pid"), + "OBJECT_SYSTEMD_OWNER_UID": integer("journald.object.systemd.owner_uid"), + "OBJECT_SYSTEMD_SESSION": text("journald.object.systemd.session"), + "OBJECT_SYSTEMD_UNIT": text("journald.object.systemd.unit"), + "OBJECT_SYSTEMD_USER_UNIT": text("journald.object.systemd.user_unit"), + "OBJECT_UID": integer("journald.object.uid"), + "_KERNEL_DEVICE": text("journald.kernel.device"), + "_KERNEL_SUBSYSTEM": text("journald.kernel.subsystem"), + "_SYSTEMD_INVOCATION_ID": text("systemd.invocation_id"), + "_SYSTEMD_USER_SLICE": text("systemd.user_slice"), + "_UDEV_DEVLINK": text("journald.kernel.device_symlinks"), + "_UDEV_DEVNODE": text("journald.kernel.device_node_path"), + "_UDEV_SYSNAME": text("journald.kernel.device_name"), + sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: integer("process.audit.login_uid"), + sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: text("process.audit.session"), + sdjournal.SD_JOURNAL_FIELD_BOOT_ID: text("host.boot_id"), + sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: text("process.capabilites"), + sdjournal.SD_JOURNAL_FIELD_CMDLINE: text("process.cmd"), + sdjournal.SD_JOURNAL_FIELD_CODE_FILE: text("journald.code.file"), + sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: text("journald.code.func"), + sdjournal.SD_JOURNAL_FIELD_CODE_LINE: integer("journald.code.line"), + sdjournal.SD_JOURNAL_FIELD_COMM: text("process.name"), + sdjournal.SD_JOURNAL_FIELD_EXE: text("process.executable"), + sdjournal.SD_JOURNAL_FIELD_GID: integer("process.uid"), + sdjournal.SD_JOURNAL_FIELD_HOSTNAME: text("host.hostname"), + sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: text("host.id"), + sdjournal.SD_JOURNAL_FIELD_MESSAGE: text("message"), + sdjournal.SD_JOURNAL_FIELD_PID: integer("process.pid"), + sdjournal.SD_JOURNAL_FIELD_PRIORITY: integer("syslog.priority", "log.syslog.priority"), + sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: integer("syslog.facility", "log.syslog.facility.name"), + sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: text("syslog.identifier"), + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: integer("syslog.pid"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: text("systemd.cgroup"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: integer("systemd.owner_uid"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: text("systemd.session"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: text("systemd.slice"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: text("systemd.unit"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: text("systemd.user_unit"), + sdjournal.SD_JOURNAL_FIELD_TRANSPORT: text("systemd.transport"), + sdjournal.SD_JOURNAL_FIELD_UID: integer("process.uid"), + + // docker journald fields from: https://docs.docker.com/config/containers/logging/journald/ + "CONTAINER_ID": text("container.id_truncated"), + "CONTAINER_ID_FULL": text("container.id"), + "CONTAINER_NAME": text("container.name"), + "CONTAINER_TAG": text("container.log.tag"), + "CONTAINER_PARTIAL_MESSAGE": text("container.partial"), + + // dropped fields + sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: ignoredField, // saved in the registry + sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: ignoredField, // saved in the registry + sdjournal.SD_JOURNAL_FIELD_CURSOR: ignoredField, // saved in the registry + "_SOURCE_MONOTONIC_TIMESTAMP": ignoredField, // received timestamp stored in @timestamp +} diff --git a/journalbeat/pkg/journalfield/default_other.go b/journalbeat/pkg/journalfield/default_other.go new file mode 100644 index 00000000000..ca3d26c9266 --- /dev/null +++ b/journalbeat/pkg/journalfield/default_other.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build !linux !cgo + +package journalfield + +// journaldEventFields provides default field mappings and conversions rules. +var journaldEventFields = FieldConversion{ + // provided by systemd journal + "COREDUMP_UNIT": text("journald.coredump.unit"), + "COREDUMP_USER_UNIT": text("journald.coredump.user_unit"), + "OBJECT_AUDIT_LOGINUID": integer("journald.object.audit.login_uid"), + "OBJECT_AUDIT_SESSION": integer("journald.object.audit.session"), + "OBJECT_CMDLINE": text("journald.object.cmd"), + "OBJECT_COMM": text("journald.object.name"), + "OBJECT_EXE": text("journald.object.executable"), + "OBJECT_GID": integer("journald.object.gid"), + "OBJECT_PID": integer("journald.object.pid"), + "OBJECT_SYSTEMD_OWNER_UID": integer("journald.object.systemd.owner_uid"), + "OBJECT_SYSTEMD_SESSION": text("journald.object.systemd.session"), + "OBJECT_SYSTEMD_UNIT": text("journald.object.systemd.unit"), + "OBJECT_SYSTEMD_USER_UNIT": text("journald.object.systemd.user_unit"), + "OBJECT_UID": integer("journald.object.uid"), + "_KERNEL_DEVICE": text("journald.kernel.device"), + "_KERNEL_SUBSYSTEM": text("journald.kernel.subsystem"), + "_SYSTEMD_INVOCATION_ID": text("systemd.invocation_id"), + "_SYSTEMD_USER_SLICE": text("systemd.user_slice"), + "_UDEV_DEVLINK": text("journald.kernel.device_symlinks"), + "_UDEV_DEVNODE": text("journald.kernel.device_node_path"), + "_UDEV_SYSNAME": text("journald.kernel.device_name"), +} diff --git a/journalbeat/pkg/journalfield/matcher.go b/journalbeat/pkg/journalfield/matcher.go new file mode 100644 index 00000000000..0227b2aae83 --- /dev/null +++ b/journalbeat/pkg/journalfield/matcher.go @@ -0,0 +1,120 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalfield + +import ( + "fmt" + "strings" +) + +// Matcher is a single field condition for filtering journal entries. +// +// The Matcher type can be used as is with Beats configuration unpacking. The +// internal default conversion table will be used, similar to BuildMatcher. +type Matcher struct { + str string +} + +// MatcherBuilder can be used to create a custom builder for creating matchers +// based on a conversion table. +type MatcherBuilder struct { + Conversions map[string]Conversion +} + +type journal interface { + AddMatch(string) error + AddDisjunction() error +} + +var defaultBuilder = MatcherBuilder{Conversions: journaldEventFields} + +// Build creates a new Matcher using the configured conversion table. +// If no table has been configured the internal default table will be used. +func (b MatcherBuilder) Build(in string) (Matcher, error) { + elems := strings.Split(in, "=") + if len(elems) != 2 { + return Matcher{}, fmt.Errorf("invalid match format: %s", in) + } + + conversions := b.Conversions + if conversions == nil { + conversions = journaldEventFields + } + + for journalKey, eventField := range conversions { + for _, name := range eventField.Names { + if elems[0] == name { + return Matcher{journalKey + "=" + elems[1]}, nil + } + } + } + + // pass custom fields as is + return Matcher{in}, nil +} + +// BuildMatcher creates a Matcher from a field filter string. +func BuildMatcher(in string) (Matcher, error) { + return defaultBuilder.Build(in) +} + +// IsValid returns true if the matcher was initialized correctly. +func (m Matcher) IsValid() bool { return m.str != "" } + +// String returns the string representation of the field match. +func (m Matcher) String() string { return m.str } + +// Apply adds the field match to an open journal for filtering. +func (m Matcher) Apply(j journal) error { + if !m.IsValid() { + return fmt.Errorf("can not apply invalid matcher to a journal") + } + + err := j.AddMatch(m.str) + if err != nil { + return fmt.Errorf("error adding match '%s' to journal: %v", m.str, err) + } + return nil +} + +// Unpack initializes the Matcher from a given string representation. Unpack +// fails if the input string is invalid. +// Unpack can be used with Beats configuration loading. +func (m *Matcher) Unpack(value string) error { + tmp, err := BuildMatcher(value) + if err != nil { + return err + } + *m = tmp + return nil +} + +// ApplyMatchersOr adds a list of matchers to a journal, calling AddDisjunction after each matcher being added. +func ApplyMatchersOr(j journal, matchers []Matcher) error { + for _, m := range matchers { + if err := m.Apply(j); err != nil { + return err + } + + if err := j.AddDisjunction(); err != nil { + return fmt.Errorf("error adding disjunction to journal: %v", err) + } + } + + return nil +} diff --git a/journalbeat/pkg/journalfield/matcher_test.go b/journalbeat/pkg/journalfield/matcher_test.go new file mode 100644 index 00000000000..9ce8c63c8a4 --- /dev/null +++ b/journalbeat/pkg/journalfield/matcher_test.go @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build linux,cgo + +package journalfield + +import ( + "testing" + + "github.com/coreos/go-systemd/v22/sdjournal" +) + +func TestApplyMatchersOr(t *testing.T) { + cases := map[string]struct { + filters []string + wantErr bool + }{ + "correct filter expression": { + filters: []string{"systemd.unit=nginx"}, + wantErr: false, + }, + "custom field": { + filters: []string{"_MY_CUSTOM_FIELD=value"}, + wantErr: false, + }, + "mixed filters": { + filters: []string{"systemd.unit=nginx", "_MY_CUSTOM_FIELD=value"}, + wantErr: false, + }, + "same field filters": { + filters: []string{"systemd.unit=nginx", "systemd.unit=mysql"}, + wantErr: false, + }, + "incorrect separator": { + filters: []string{"systemd.unit~nginx"}, + wantErr: true, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + journal, err := sdjournal.NewJournal() + if err != nil { + t.Fatalf("error while creating test journal: %v", err) + } + defer journal.Close() + + matchers := make([]Matcher, len(test.filters)) + for i, str := range test.filters { + m, err := BuildMatcher(str) + if err != nil && !test.wantErr { + t.Fatalf("unexpected error compiling the filters: %v", err) + } + matchers[i] = m + } + + // double check if journald likes our filters + err = ApplyMatchersOr(journal, matchers) + fail := (test.wantErr && err == nil) || (!test.wantErr && err != nil) + if fail { + t.Errorf("unexpected outcome: error: '%v', expected error: %v", err, test.wantErr) + } + }) + } +} diff --git a/journalbeat/pkg/journalread/mode.go b/journalbeat/pkg/journalread/mode.go new file mode 100644 index 00000000000..3eff2d93c5d --- /dev/null +++ b/journalbeat/pkg/journalread/mode.go @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalread + +import ( + "errors" + "fmt" +) + +// SeekMode is used by (*Reader).Seek to decide where to advance the read pointer to. +type SeekMode uint + +const ( + // SeekInvalid is an invalid value for seek + SeekInvalid SeekMode = iota + // SeekHead option seeks to the head of a journal + SeekHead + // SeekTail option seeks to the tail of a journal + SeekTail + // SeekCursor option seeks to the position specified in the cursor + SeekCursor +) + +var seekModes = map[string]SeekMode{ + "head": SeekHead, + "tail": SeekTail, + "cursor": SeekCursor, +} + +var errInvalidSeekFallback = errors.New("invalid setting for cursor_seek_fallback") + +// Unpack validates and unpack "seek" config options. It returns an error if +// the string is no valid seek mode. +func (m *SeekMode) Unpack(value string) error { + mode, ok := seekModes[value] + if !ok { + return fmt.Errorf("invalid seek mode '%s'", value) + } + + *m = mode + return nil +} diff --git a/journalbeat/pkg/journalread/mode_test.go b/journalbeat/pkg/journalread/mode_test.go new file mode 100644 index 00000000000..aef0ed4150c --- /dev/null +++ b/journalbeat/pkg/journalread/mode_test.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalread + +import ( + "testing" +) + +func TestMode_Unpack(t *testing.T) { + t.Run("ok", func(t *testing.T) { + tests := map[string]SeekMode{ + "head": SeekHead, + "tail": SeekTail, + "cursor": SeekCursor, + } + + for str, want := range tests { + t.Run(str, func(t *testing.T) { + var m SeekMode + if err := m.Unpack(str); err != nil { + t.Fatal(err) + } + + if m != want { + t.Errorf("wrong mode, expected %v, got %v", want, m) + } + }) + } + }) + + t.Run("failing", func(t *testing.T) { + cases := []string{"invalid", "", "unknown"} + + for _, str := range cases { + t.Run(str, func(t *testing.T) { + var m SeekMode + if err := m.Unpack(str); err == nil { + t.Errorf("an error was expected, got %v", m) + } + }) + } + }) +} diff --git a/journalbeat/pkg/journalread/reader.go b/journalbeat/pkg/journalread/reader.go new file mode 100644 index 00000000000..7d5ac50e965 --- /dev/null +++ b/journalbeat/pkg/journalread/reader.go @@ -0,0 +1,199 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux,cgo + +package journalread + +import ( + "fmt" + "io" + "os" + "syscall" + "time" + + "github.com/coreos/go-systemd/v22/sdjournal" + "github.com/urso/sderr" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common/backoff" + "github.com/elastic/beats/v7/libbeat/common/cleanup" + "github.com/elastic/beats/v7/libbeat/logp" +) + +// Reader implements a Journald base reader with backoff support. The reader +// will block until a new entry can be read from the journal. +type Reader struct { + log *logp.Logger + backoff backoff.Backoff + journal journal +} + +type journal interface { + Close() error + + Next() (uint64, error) + Wait(time.Duration) int + GetEntry() (*sdjournal.JournalEntry, error) + + SeekHead() error + SeekTail() error + SeekCursor(string) error +} + +// LocalSystemJournalID is the ID of the local system journal. +const localSystemJournalID = "LOCAL_SYSTEM_JOURNAL" + +// NewReader creates a new Reader for an already opened journal. The reader assumed to take +// ownership of the journal, and needs to be closed. +func NewReader(log *logp.Logger, journal journal, backoff backoff.Backoff) *Reader { + return &Reader{log: log, journal: journal, backoff: backoff} +} + +// Open opens a journal and creates a reader for it. +// Additonal settings can be applied to the journal by passing functions to with. +// Open returns an error if the journal can not be opened, or if one with-function failed. +// +// Open will opend the systems journal if the path is empty or matches LOCAL_SYSTEM_JOURNAL. +// The path can optionally point to a file or a directory. +func Open(log *logp.Logger, path string, backoff backoff.Backoff, with ...func(j *sdjournal.Journal) error) (*Reader, error) { + j, err := openJournal(path) + if err != nil { + return nil, err + } + + ok := false + defer cleanup.IfNot(&ok, func() { j.Close() }) + + for _, w := range with { + if err := w(j); err != nil { + return nil, err + } + } + + ok = true + return NewReader(log, j, backoff), nil +} + +func openJournal(path string) (*sdjournal.Journal, error) { + if path == localSystemJournalID || path == "" { + j, err := sdjournal.NewJournal() + if err != nil { + err = sderr.Wrap(err, "failed to open local journal") + } + return j, err + } + + stat, err := os.Stat(path) + if err != nil { + return nil, sderr.Wrap(err, "failed to read meta data for %{path}", path) + } + + if stat.IsDir() { + j, err := sdjournal.NewJournalFromDir(path) + if err != nil { + err = sderr.Wrap(err, "failed to open journal directory %{path}", path) + } + return j, err + } + + j, err := sdjournal.NewJournalFromFiles(path) + if err != nil { + err = sderr.Wrap(err, "failed to open journal file %{path}", path) + } + return j, err +} + +// Close closes the journal. +func (r *Reader) Close() error { + return r.journal.Close() +} + +// Seek moves the read pointer to a new position. +// If a cursor or SeekTail is given, Seek tries to ignore the entry at the +// given position, jumping right to the next entry. +func (r *Reader) Seek(mode SeekMode, cursor string) (err error) { + switch mode { + case SeekHead: + err = r.journal.SeekHead() + case SeekTail: + if err = r.journal.SeekTail(); err == nil { + _, err = r.journal.Next() + } + case SeekCursor: + if err = r.journal.SeekCursor(cursor); err == nil { + _, err = r.journal.Next() + } + default: + return fmt.Errorf("invalid seek mode '%v'", mode) + } + return err +} + +// Next reads a new journald entry from the journal. It blocks if there is +// currently no entry available in the journal, or until an error has occured. +func (r *Reader) Next(cancel input.Canceler) (*sdjournal.JournalEntry, error) { + for cancel.Err() == nil { + c, err := r.journal.Next() + if err != nil && err != io.EOF { + return nil, err + } + + switch { + // error while reading next entry + case c < 0: + return nil, fmt.Errorf("error while reading next entry %+v", syscall.Errno(-c)) + // no new entry, so wait + case c == 0: + hasNewEntry, err := r.checkForNewEvents() + if err != nil { + return nil, err + } + if !hasNewEntry { + // TODO: backoff support is currently not cancellable :( + r.backoff.Wait() + } + continue + // new entries are available + default: + } + + entry, err := r.journal.GetEntry() + if err != nil { + return nil, err + } + r.backoff.Reset() + + return entry, nil + } + return nil, cancel.Err() +} + +func (r *Reader) checkForNewEvents() (bool, error) { + c := r.journal.Wait(100 * time.Millisecond) + switch c { + case sdjournal.SD_JOURNAL_NOP: + return false, nil + // new entries are added or the journal has changed (e.g. vacuum, rotate) + case sdjournal.SD_JOURNAL_APPEND, sdjournal.SD_JOURNAL_INVALIDATE: + return true, nil + default: + } + + r.log.Errorf("Unknown return code from Wait: %d\n", c) + return false, nil +} diff --git a/journalbeat/reader/config.go b/journalbeat/reader/config.go index f8dbabe5fee..e559681aea7 100644 --- a/journalbeat/reader/config.go +++ b/journalbeat/reader/config.go @@ -20,7 +20,8 @@ package reader import ( "time" - "github.com/elastic/beats/v7/journalbeat/config" + "github.com/elastic/beats/v7/journalbeat/pkg/journalfield" + "github.com/elastic/beats/v7/journalbeat/pkg/journalread" ) // Config stores the options of a reder. @@ -29,16 +30,16 @@ type Config struct { Path string // Seek specifies the seeking stategy. // Possible values: head, tail, cursor. - Seek config.SeekMode + Seek journalread.SeekMode // CursorSeekFallback sets where to seek if registry file is not available. - CursorSeekFallback config.SeekMode + CursorSeekFallback journalread.SeekMode // MaxBackoff is the limit of the backoff time. MaxBackoff time.Duration // Backoff is the current interval to wait before // attemting to read again from the journal. Backoff time.Duration // Matches store the key value pairs to match entries. - Matches []string + Matches []journalfield.Matcher // SaveRemoteHostname defines if the original source of the entry needs to be saved. SaveRemoteHostname bool // CheckpointID is the identifier to use when persisting state. diff --git a/journalbeat/reader/fields.go b/journalbeat/reader/fields.go index d0230849cba..0b6efc3cdc8 100644 --- a/journalbeat/reader/fields.go +++ b/journalbeat/reader/fields.go @@ -18,78 +18,3 @@ //+build linux,cgo package reader - -import "github.com/coreos/go-systemd/v22/sdjournal" - -type fieldConversion struct { - names []string - isInteger bool - dropped bool -} - -var ( - journaldEventFields = map[string]fieldConversion{ - // provided by systemd journal - "COREDUMP_UNIT": fieldConversion{[]string{"journald.coredump.unit"}, false, false}, - "COREDUMP_USER_UNIT": fieldConversion{[]string{"journald.coredump.user_unit"}, false, false}, - "OBJECT_AUDIT_LOGINUID": fieldConversion{[]string{"journald.object.audit.login_uid"}, true, false}, - "OBJECT_AUDIT_SESSION": fieldConversion{[]string{"journald.object.audit.session"}, true, false}, - "OBJECT_CMDLINE": fieldConversion{[]string{"journald.object.cmd"}, false, false}, - "OBJECT_COMM": fieldConversion{[]string{"journald.object.name"}, false, false}, - "OBJECT_EXE": fieldConversion{[]string{"journald.object.executable"}, false, false}, - "OBJECT_GID": fieldConversion{[]string{"journald.object.gid"}, true, false}, - "OBJECT_PID": fieldConversion{[]string{"journald.object.pid"}, true, false}, - "OBJECT_SYSTEMD_OWNER_UID": fieldConversion{[]string{"journald.object.systemd.owner_uid"}, true, false}, - "OBJECT_SYSTEMD_SESSION": fieldConversion{[]string{"journald.object.systemd.session"}, false, false}, - "OBJECT_SYSTEMD_UNIT": fieldConversion{[]string{"journald.object.systemd.unit"}, false, false}, - "OBJECT_SYSTEMD_USER_UNIT": fieldConversion{[]string{"journald.object.systemd.user_unit"}, false, false}, - "OBJECT_UID": fieldConversion{[]string{"journald.object.uid"}, true, false}, - "_KERNEL_DEVICE": fieldConversion{[]string{"journald.kernel.device"}, false, false}, - "_KERNEL_SUBSYSTEM": fieldConversion{[]string{"journald.kernel.subsystem"}, false, false}, - "_SYSTEMD_INVOCATION_ID": fieldConversion{[]string{"systemd.invocation_id"}, false, false}, - "_SYSTEMD_USER_SLICE": fieldConversion{[]string{"systemd.user_slice"}, false, false}, - "_UDEV_DEVLINK": fieldConversion{[]string{"journald.kernel.device_symlinks"}, false, false}, // TODO aggregate multiple elements - "_UDEV_DEVNODE": fieldConversion{[]string{"journald.kernel.device_node_path"}, false, false}, - "_UDEV_SYSNAME": fieldConversion{[]string{"journald.kernel.device_name"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: fieldConversion{[]string{"process.audit.login_uid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: fieldConversion{[]string{"process.audit.session"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_BOOT_ID: fieldConversion{[]string{"host.boot_id"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: fieldConversion{[]string{"process.capabilites"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CMDLINE: fieldConversion{[]string{"process.cmd"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CODE_FILE: fieldConversion{[]string{"journald.code.file"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: fieldConversion{[]string{"journald.code.func"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CODE_LINE: fieldConversion{[]string{"journald.code.line"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_COMM: fieldConversion{[]string{"process.name"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_EXE: fieldConversion{[]string{"process.executable"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_GID: fieldConversion{[]string{"process.uid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_HOSTNAME: fieldConversion{[]string{"host.hostname"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: fieldConversion{[]string{"host.id"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_MESSAGE: fieldConversion{[]string{"message"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_PID: fieldConversion{[]string{"process.pid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_PRIORITY: fieldConversion{[]string{"syslog.priority", "log.syslog.priority"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: fieldConversion{[]string{"syslog.facility", "log.syslog.facility.name"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: fieldConversion{[]string{"syslog.identifier"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: fieldConversion{[]string{"syslog.pid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: fieldConversion{[]string{"systemd.cgroup"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: fieldConversion{[]string{"systemd.owner_uid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: fieldConversion{[]string{"systemd.session"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: fieldConversion{[]string{"systemd.slice"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: fieldConversion{[]string{"systemd.unit"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: fieldConversion{[]string{"systemd.user_unit"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_TRANSPORT: fieldConversion{[]string{"systemd.transport"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_UID: fieldConversion{[]string{"process.uid"}, true, false}, - - // docker journald fields from: https://docs.docker.com/config/containers/logging/journald/ - "CONTAINER_ID": fieldConversion{[]string{"container.id_truncated"}, false, false}, - "CONTAINER_ID_FULL": fieldConversion{[]string{"container.id"}, false, false}, - "CONTAINER_NAME": fieldConversion{[]string{"container.name"}, false, false}, - "CONTAINER_TAG": fieldConversion{[]string{"container.log.tag"}, false, false}, - "CONTAINER_PARTIAL_MESSAGE": fieldConversion{[]string{"container.partial"}, false, false}, - - // dropped fields - sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: fieldConversion{nil, false, true}, // saved in the registry - sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: fieldConversion{nil, false, true}, // saved in the registry - sdjournal.SD_JOURNAL_FIELD_CURSOR: fieldConversion{nil, false, true}, // saved in the registry - "_SOURCE_MONOTONIC_TIMESTAMP": fieldConversion{nil, false, true}, // received timestamp stored in @timestamp - } -) diff --git a/journalbeat/reader/journal.go b/journalbeat/reader/journal.go index 7a254f4f6d2..fb5b91c5019 100644 --- a/journalbeat/reader/journal.go +++ b/journalbeat/reader/journal.go @@ -20,28 +20,23 @@ package reader import ( - "fmt" - "io" - "os" - "strconv" - "strings" - "syscall" "time" "github.com/coreos/go-systemd/v22/sdjournal" - "github.com/pkg/errors" "github.com/elastic/beats/v7/journalbeat/checkpoint" "github.com/elastic/beats/v7/journalbeat/cmd/instance" - "github.com/elastic/beats/v7/journalbeat/config" + "github.com/elastic/beats/v7/journalbeat/pkg/journalfield" + "github.com/elastic/beats/v7/journalbeat/pkg/journalread" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/backoff" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/ctxtool" ) // Reader reads entries from journal(s). type Reader struct { + r *journalread.Reader journal *sdjournal.Journal config Config done chan struct{} @@ -51,223 +46,83 @@ type Reader struct { // New creates a new journal reader and moves the FP to the configured position. func New(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) { - f, err := os.Stat(c.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to open file") - } - - var j *sdjournal.Journal - if f.IsDir() { - j, err = sdjournal.NewJournalFromDir(c.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to open journal directory") - } - } else { - j, err = sdjournal.NewJournalFromFiles(c.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to open journal file") - } - } - - l := logger.With("path", c.Path) - l.Debug("New journal is opened for reading") - - return newReader(l, done, c, j, state) + return newReader(c.Path, c, done, state, logger) } // NewLocal creates a reader to read form the local journal and moves the FP // to the configured position. func NewLocal(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) { - j, err := sdjournal.NewJournal() - if err != nil { - return nil, errors.Wrap(err, "failed to open local journal") - } - - l := logger.With("path", "local") - l.Debug("New local journal is opened for reading") - - return newReader(l, done, c, j, state) + return newReader(LocalSystemJournalID, c, done, state, logger) } -func newReader(logger *logp.Logger, done chan struct{}, c Config, journal *sdjournal.Journal, state checkpoint.JournalState) (*Reader, error) { - err := setupMatches(journal, c.Matches) +func newReader(path string, c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) { + logger = logger.With("path", path) + backoff := backoff.NewExpBackoff(done, c.Backoff, c.MaxBackoff) + + var journal *sdjournal.Journal + r, err := journalread.Open(logger, c.Path, backoff, func(j *sdjournal.Journal) error { + journal = j + return journalfield.ApplyMatchersOr(j, c.Matches) + }) if err != nil { return nil, err } - r := &Reader{ - journal: journal, - config: c, - done: done, - logger: logger, - backoff: backoff.NewExpBackoff(done, c.Backoff, c.MaxBackoff), + if err := r.Seek(seekBy(logger, c, state)); err != nil { + logger.Error("Continue from current position. Seek failed with: %v", err) } - r.seek(state.Cursor) + logger.Debug("New journal is opened for reading") instance.AddJournalToMonitor(c.Path, journal) - return r, nil + return &Reader{ + journal: journal, + config: c, + done: done, + logger: logger, + backoff: backoff, + }, nil } -func setupMatches(j *sdjournal.Journal, matches []string) error { - for _, m := range matches { - elems := strings.Split(m, "=") - if len(elems) != 2 { - return fmt.Errorf("invalid match format: %s", m) - } - - var p string - for journalKey, eventField := range journaldEventFields { - for _, name := range eventField.names { - if elems[0] == name { - p = journalKey + "=" + elems[1] - } - } - } - - // pass custom fields as is - if p == "" { - p = m - } - - logp.Debug("journal", "Added matcher expression: %s", p) - - err := j.AddMatch(p) - if err != nil { - return fmt.Errorf("error adding match to journal %v", err) - } - - err = j.AddDisjunction() - if err != nil { - return fmt.Errorf("error adding disjunction to journal: %v", err) +func seekBy(log *logp.Logger, c Config, state checkpoint.JournalState) (journalread.SeekMode, string) { + mode := c.Seek + if mode == journalread.SeekCursor && state.Cursor == "" { + mode = c.CursorSeekFallback + if mode != journalread.SeekHead && mode != journalread.SeekTail { + log.Error("Invalid option for cursor_seek_fallback") + mode = journalread.SeekHead } } - return nil + return mode, state.Cursor } -// seek seeks to the position determined by the coniguration and cursor state. -func (r *Reader) seek(cursor string) { - switch r.config.Seek { - case config.SeekCursor: - if cursor == "" { - switch r.config.CursorSeekFallback { - case config.SeekHead: - r.journal.SeekHead() - r.logger.Debug("Seeking method set to cursor, but no state is saved for reader. Starting to read from the beginning") - case config.SeekTail: - r.journal.SeekTail() - r.journal.Next() - r.logger.Debug("Seeking method set to cursor, but no state is saved for reader. Starting to read from the end") - default: - r.logger.Error("Invalid option for cursor_seek_fallback") - } - return - } - r.journal.SeekCursor(cursor) - _, err := r.journal.Next() - if err != nil { - r.logger.Error("Error while seeking to cursor") - } - r.logger.Debug("Seeked to position defined in cursor") - case config.SeekTail: - r.journal.SeekTail() - r.journal.Next() - r.logger.Debug("Tailing the journal file") - case config.SeekHead: - r.journal.SeekHead() - r.logger.Debug("Reading from the beginning of the journal file") - default: - r.logger.Error("Invalid seeking mode") - } +// Close closes the underlying journal reader. +func (r *Reader) Close() { + instance.StopMonitoringJournal(r.config.Path) + r.r.Close() } // Next waits until a new event shows up and returns it. // It blocks until an event is returned or an error occurs. func (r *Reader) Next() (*beat.Event, error) { - for { - select { - case <-r.done: - return nil, nil - default: - } - - c, err := r.journal.Next() - if err != nil && err != io.EOF { - return nil, err - } - - switch { - // error while reading next entry - case c < 0: - return nil, fmt.Errorf("error while reading next entry %+v", syscall.Errno(-c)) - // no new entry, so wait - case c == 0: - hasNewEntry, err := r.checkForNewEvents() - if err != nil { - return nil, err - } - if !hasNewEntry { - r.backoff.Wait() - } - continue - // new entries are available - default: - } - - entry, err := r.journal.GetEntry() - if err != nil { - return nil, err - } - event := r.toEvent(entry) - r.backoff.Reset() - - return event, nil - } -} - -func (r *Reader) checkForNewEvents() (bool, error) { - c := r.journal.Wait(100 * time.Millisecond) - switch c { - case sdjournal.SD_JOURNAL_NOP: - return false, nil - // new entries are added or the journal has changed (e.g. vacuum, rotate) - case sdjournal.SD_JOURNAL_APPEND, sdjournal.SD_JOURNAL_INVALIDATE: - return true, nil - default: + entry, err := r.r.Next(ctxtool.FromChannel(r.done)) + if err != nil { + return nil, err } - r.logger.Errorf("Unknown return code from Wait: %d\n", c) - return false, nil + event := toEvent(r.logger, r.config.CheckpointID, entry, r.config.SaveRemoteHostname) + return event, nil } // toEvent creates a beat.Event from journal entries. -func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { - fields := common.MapStr{ - "event": common.MapStr{ - "kind": "event", - }, - } - custom := common.MapStr{} - - for entryKey, v := range entry.Fields { - if fieldConversionInfo, ok := journaldEventFields[entryKey]; !ok { - normalized := strings.ToLower(strings.TrimLeft(entryKey, "_")) - custom.Put(normalized, v) - } else if !fieldConversionInfo.dropped { - value := r.convertNamedField(fieldConversionInfo, v) - for _, name := range fieldConversionInfo.names { - fields.Put(name, value) - } - } - } - - if len(custom) != 0 { - fields.Put("journald.custom", custom) - } +func toEvent(logger *logp.Logger, id string, entry *sdjournal.JournalEntry, saveRemoteHostname bool) *beat.Event { + created := time.Now() + fields := journalfield.NewConverter(logger, nil).Convert(entry.Fields) + fields.Put("event.kind", "event") // if entry is coming from a remote journal, add_host_metadata overwrites the source hostname, so it // has to be copied to a different field - if r.config.SaveRemoteHostname { + if saveRemoteHostname { remoteHostname, err := fields.GetValue("host.hostname") if err == nil { fields.Put("log.source.address", remoteHostname) @@ -275,13 +130,13 @@ func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { } state := checkpoint.JournalState{ - Path: r.config.CheckpointID, + Path: id, Cursor: entry.Cursor, RealtimeTimestamp: entry.RealtimeTimestamp, MonotonicTimestamp: entry.MonotonicTimestamp, } - fields.Put("event.created", time.Now()) + fields.Put("event.created", created) receivedByJournal := time.Unix(0, int64(entry.RealtimeTimestamp)*1000) event := beat.Event{ @@ -291,29 +146,3 @@ func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { } return &event } - -func (r *Reader) convertNamedField(fc fieldConversion, value string) interface{} { - if fc.isInteger { - v, err := strconv.ParseInt(value, 10, 64) - if err != nil { - // On some versions of systemd the 'syslog.pid' can contain the username - // appended to the end of the pid. In most cases this does not occur - // but in the cases that it does, this tries to strip ',\w*' from the - // value and then perform the conversion. - s := strings.Split(value, ",") - v, err = strconv.ParseInt(s[0], 10, 64) - if err != nil { - r.logger.Debugf("Failed to convert field: %v \"%v\" to int: %v", fc.names, value, err) - return value - } - } - return v - } - return value -} - -// Close closes the underlying journal reader. -func (r *Reader) Close() { - instance.StopMonitoringJournal(r.config.Path) - r.journal.Close() -} diff --git a/journalbeat/reader/journal_test.go b/journalbeat/reader/journal_test.go deleted file mode 100644 index 4293b4b31e1..00000000000 --- a/journalbeat/reader/journal_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//+build linux,cgo - -package reader - -import ( - "reflect" - "testing" - - "github.com/coreos/go-systemd/v22/sdjournal" - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/journalbeat/checkpoint" - "github.com/elastic/beats/v7/journalbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -type ToEventTestCase struct { - entry sdjournal.JournalEntry - expectedFields common.MapStr -} - -type SetupMatchesTestCase struct { - matches []string - expectError bool -} - -func TestToEvent(t *testing.T) { - tests := []ToEventTestCase{ - // field name from fields.go - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_BOOT_ID: "123456", - }, - }, - expectedFields: common.MapStr{ - "host": common.MapStr{ - "boot_id": "123456", - }, - }, - }, - // 'syslog.pid' field without user append - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "pid": int64(123456), - }, - }, - }, - // 'syslog.priority' field with junk - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_PRIORITY: "123456, ", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "priority": int64(123456), - }, - "log": common.MapStr{ - "syslog": common.MapStr{ - "priority": int64(123456), - }, - }, - }, - }, - // 'syslog.pid' field with user append - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456,root", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "pid": int64(123456), - }, - }, - }, - // 'syslog.pid' field empty - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "pid": "", - }, - }, - }, - // custom field - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - "my_custom_field": "value", - }, - }, - expectedFields: common.MapStr{ - "journald": common.MapStr{ - "custom": common.MapStr{ - "my_custom_field": "value", - }, - }, - }, - }, - // dropped field - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - "_SOURCE_MONOTONIC_TIMESTAMP": "value", - }, - }, - expectedFields: common.MapStr{}, - }, - } - - instance.SetupJournalMetrics() - r, err := NewLocal(Config{Path: "dummy.journal"}, nil, checkpoint.JournalState{}, logp.NewLogger("test")) - if err != nil { - t.Fatalf("error creating test journal: %v", err) - } - for _, test := range tests { - event := r.toEvent(&test.entry) - event.Fields.Delete("event") - assert.True(t, reflect.DeepEqual(event.Fields, test.expectedFields)) - } -} - -func TestSetupMatches(t *testing.T) { - tests := []SetupMatchesTestCase{ - // correct filter expression - SetupMatchesTestCase{ - matches: []string{"systemd.unit=nginx"}, - expectError: false, - }, - // custom field - SetupMatchesTestCase{ - matches: []string{"_MY_CUSTOM_FIELD=value"}, - expectError: false, - }, - // incorrect separator - SetupMatchesTestCase{ - matches: []string{"systemd.unit~nginx"}, - expectError: true, - }, - } - journal, err := sdjournal.NewJournal() - if err != nil { - t.Fatalf("error while creating test journal: %v", err) - } - - for _, test := range tests { - err = setupMatches(journal, test.matches) - if err != nil && !test.expectError { - t.Errorf("unexpected outcome of setupMatches: error: '%v', expected error: %v", err, test.expectError) - } - } -} diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 649c372bfd6..30ed913060b 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -284,6 +284,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet var ( annotations = common.MapStr{} nsAnn = common.MapStr{} + events = make([]bus.Event, 0) ) for k, v := range pod.GetObjectMeta().GetAnnotations() { safemapstr.Put(annotations, k, v) @@ -299,7 +300,6 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet } } - emitted := 0 // Emit container and port information for _, c := range containers { // If it doesn't have an ID, container doesn't exist in @@ -345,8 +345,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet "kubernetes": meta, }, } - p.publish(event) - emitted++ + events = append(events, event) } for _, port := range c.Ports { @@ -361,16 +360,16 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet "kubernetes": meta, }, } - p.publish(event) - emitted++ + events = append(events, event) } } - // Finally publish a pod level event so that hints that have no exposed ports can get processed. + // Publish a pod level event so that hints that have no exposed ports can get processed. // Log hints would just ignore this event as there is no ${data.container.id} - // Publish the pod level hint only if atleast one container level hint was emitted. This ensures that there is + // Publish the pod level hint only if at least one container level hint was generated. This ensures that there is // no unnecessary pod level events emitted prematurely. - if emitted != 0 { + // We publish the pod level hint first so that it doesn't override a valid container level event. + if len(events) != 0 { meta := p.metagen.Generate(pod) // Information that can be used in discovering a workload @@ -392,6 +391,10 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet }, } p.publish(event) + } + // Publish the container level hints finally. + for _, event := range events { + p.publish(event) } } diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 7e5d51b23b0..2dacee2b9fc 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -353,7 +353,7 @@ func TestEmitEvent(t *testing.T) { Message string Flag string Pod *kubernetes.Pod - Expected bus.Event + Expected []bus.Event }{ { Message: "Test common pod start", @@ -389,44 +389,227 @@ func TestEmitEvent(t *testing.T) { }, }, }, - Expected: bus.Event{ - "start": true, - "host": "127.0.0.1", - "port": 0, - "id": cid, - "provider": UUID, - "kubernetes": common.MapStr{ - "container": common.MapStr{ - "id": "foobar", - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - "runtime": "docker", + Expected: []bus.Event{ + { + "start": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "pod": common.MapStr{ - "name": "filebeat", - "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, - "node": common.MapStr{ - "name": "node", + "config": []*common.Config{}, + }, + { + "start": true, + "host": "127.0.0.1", + "port": 0, + "id": cid, + "provider": UUID, + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "id": "foobar", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "docker", + }, + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "namespace": "default", - "annotations": common.MapStr{}, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + }, + }, + { + Message: "Test common pod start with multiple ports exposed", + Flag: "start", + Pod: &kubernetes.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, }, - "meta": common.MapStr{ + TypeMeta: typeMeta, + Status: v1.PodStatus{ + PodIP: podIP, + ContainerStatuses: []kubernetes.PodContainerStatus{ + { + Name: name, + ContainerID: containerID, + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + Spec: v1.PodSpec{ + NodeName: node, + Containers: []kubernetes.Container{ + { + Image: containerImage, + Name: name, + Ports: []v1.ContainerPort{ + { + ContainerPort: 8080, + }, + { + ContainerPort: 9090, + }, + }, + }, + }, + }, + }, + Expected: []bus.Event{ + { + "start": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "start": true, + "host": "127.0.0.1", + "port": int32(8080), + "id": cid, + "provider": UUID, + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "id": "foobar", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "docker", + }, + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "start": true, + "host": "127.0.0.1", + "port": int32(9090), + "id": cid, + "provider": UUID, "kubernetes": common.MapStr{ - "namespace": "default", "container": common.MapStr{ - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - }, "pod": common.MapStr{ + "id": "foobar", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "docker", + }, + "pod": common.MapStr{ "name": "filebeat", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - }, "node": common.MapStr{ + }, + "node": common.MapStr{ "name": "node", }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, + "config": []*common.Config{}, }, - "config": []*common.Config{}, }, }, { @@ -522,44 +705,75 @@ func TestEmitEvent(t *testing.T) { }, }, }, - Expected: bus.Event{ - "stop": true, - "host": "", - "id": cid, - "port": 0, - "provider": UUID, - "kubernetes": common.MapStr{ - "container": common.MapStr{ - "id": "", - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - "runtime": "", - }, - "pod": common.MapStr{ - "name": "filebeat", - "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + Expected: []bus.Event{ + { + "stop": true, + "host": "", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "node": common.MapStr{ - "name": "node", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, - "namespace": "default", - "annotations": common.MapStr{}, + "config": []*common.Config{}, }, - "meta": common.MapStr{ + { + "stop": true, + "host": "", + "id": cid, + "port": 0, + "provider": UUID, "kubernetes": common.MapStr{ - "namespace": "default", "container": common.MapStr{ - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - }, "pod": common.MapStr{ + "id": "", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "", + }, + "pod": common.MapStr{ "name": "filebeat", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - }, "node": common.MapStr{ + }, + "node": common.MapStr{ "name": "node", }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, + "config": []*common.Config{}, }, - "config": []*common.Config{}, }, }, { @@ -592,44 +806,176 @@ func TestEmitEvent(t *testing.T) { }, }, }, - Expected: bus.Event{ - "stop": true, - "host": "127.0.0.1", - "port": 0, - "id": cid, - "provider": UUID, - "kubernetes": common.MapStr{ - "container": common.MapStr{ - "id": "", - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - "runtime": "", + Expected: []bus.Event{ + { + "stop": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "pod": common.MapStr{ - "name": "filebeat", - "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "stop": true, + "host": "127.0.0.1", + "port": 0, + "id": cid, + "provider": UUID, + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "id": "", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "", + }, + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "node": common.MapStr{ - "name": "node", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, - "namespace": "default", - "annotations": common.MapStr{}, + "config": []*common.Config{}, }, - "meta": common.MapStr{ + }, + }, + { + Message: "Test stop pod without container id", + Flag: "stop", + Pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + TypeMeta: typeMeta, + Status: v1.PodStatus{ + PodIP: podIP, + ContainerStatuses: []kubernetes.PodContainerStatus{ + { + Name: name, + }, + }, + }, + Spec: v1.PodSpec{ + NodeName: node, + Containers: []kubernetes.Container{ + { + Image: containerImage, + Name: name, + }, + }, + }, + }, + Expected: []bus.Event{ + { + "stop": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "stop": true, + "host": "127.0.0.1", + "port": 0, + "id": cid, + "provider": UUID, "kubernetes": common.MapStr{ - "namespace": "default", "container": common.MapStr{ - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - }, "pod": common.MapStr{ + "id": "", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "", + }, + "pod": common.MapStr{ "name": "filebeat", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - }, "node": common.MapStr{ + }, + "node": common.MapStr{ "name": "node", }, + "namespace": "default", + "annotations": common.MapStr{}, }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, }, - "config": []*common.Config{}, }, }, } @@ -663,14 +1009,17 @@ func TestEmitEvent(t *testing.T) { pod.emit(test.Pod, test.Flag) - select { - case event := <-listener.Events(): - assert.Equal(t, test.Expected, event, test.Message) - case <-time.After(2 * time.Second): - if test.Expected != nil { - t.Fatal("Timeout while waiting for event") + for i := 0; i < len(test.Expected); i++ { + select { + case event := <-listener.Events(): + assert.Equal(t, test.Expected[i], event, test.Message) + case <-time.After(2 * time.Second): + if test.Expected != nil { + t.Fatal("Timeout while waiting for event") + } } } + }) } } diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 0e8e369ff5f..14ea85cd070 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -18,6 +18,7 @@ package file import ( + "fmt" "os" "path/filepath" "strconv" @@ -301,7 +302,7 @@ func (r *Rotator) openFile() error { r.file, err = os.OpenFile(r.filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, r.permissions) if err != nil { - return errors.Wrap(err, "failed to open new file") + return errors.Wrap(err, fmt.Sprintf("failed to open new file '%s'", r.filename)) } if r.redirectStderr { RedirectStandardError(r.file) diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go index 55948aacc11..3c2f2cde381 100644 --- a/libbeat/logp/config.go +++ b/libbeat/logp/config.go @@ -24,18 +24,18 @@ import ( // Config contains the configuration options for the logger. To create a Config // from a common.Config use logp/config.Build. type Config struct { - Beat string `config:",ignore"` // Name of the Beat (for default file name). - JSON bool `config:"json"` // Write logs as JSON. - Level Level `config:"level"` // Logging level (error, warning, info, debug). - Selectors []string `config:"selectors"` // Selectors for debug level logging. - ECSEnabled bool `config:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line + Beat string `config:",ignore"` // Name of the Beat (for default file name). + JSON bool `config:"json"` // Write logs as JSON. + Level Level `config:"level"` // Logging level (error, warning, info, debug). + Selectors []string `config:"selectors"` // Selectors for debug level logging. + ECSEnabled bool `config:"ecs" yaml:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line toObserver bool toIODiscard bool - ToStderr bool `config:"to_stderr"` - ToSyslog bool `config:"to_syslog"` - ToFiles bool `config:"to_files"` - ToEventLog bool `config:"to_eventlog"` + ToStderr bool `config:"to_stderr" yaml:"to_stderr"` + ToSyslog bool `config:"to_syslog" yaml:"to_syslog"` + ToFiles bool `config:"to_files" yaml:"to_files"` + ToEventLog bool `config:"to_eventlog" yaml:"to_eventlog"` Files FileConfig `config:"files"` @@ -46,14 +46,14 @@ type Config struct { // FileConfig contains the configuration options for the file output. type FileConfig struct { - Path string `config:"path"` - Name string `config:"name"` - MaxSize uint `config:"rotateeverybytes" validate:"min=1"` - MaxBackups uint `config:"keepfiles" validate:"max=1024"` + Path string `config:"path" yaml:"path"` + Name string `config:"name" yaml:"name"` + MaxSize uint `config:"rotateeverybytes" yaml:"rotateeverybytes" validate:"min=1"` + MaxBackups uint `config:"keepfiles" yaml:"keepfiles" validate:"max=1024"` Permissions uint32 `config:"permissions"` Interval time.Duration `config:"interval"` RotateOnStartup bool `config:"rotateonstartup"` - RedirectStderr bool `config:"redirect_stderr"` + RedirectStderr bool `config:"redirect_stderr" yaml:"redirect_stderr"` } const defaultLevel = InfoLevel diff --git a/libbeat/logp/level.go b/libbeat/logp/level.go index 6890192da03..95e7c5c7794 100644 --- a/libbeat/logp/level.go +++ b/libbeat/logp/level.go @@ -81,6 +81,26 @@ func (l *Level) Unpack(str string) error { return errors.Errorf("invalid level '%v'", str) } +// MarshalYAML marshals level in a correct form +func (l Level) MarshalYAML() (interface{}, error) { + s, found := levelStrings[l] + if found { + return s, nil + } + + return nil, errors.Errorf("invalid level '%d'", l) +} + +// MarshalJSON marshals level in a correct form +func (l Level) MarshalJSON() ([]byte, error) { + s, found := levelStrings[l] + if found { + return []byte(s), nil + } + + return nil, errors.Errorf("invalid level '%d'", l) +} + func (l Level) zapLevel() zapcore.Level { z, found := zapLevels[l] if found { diff --git a/libbeat/statestore/backend/memlog/memlog.go b/libbeat/statestore/backend/memlog/memlog.go index 265b53a3df1..fc93d85e85d 100644 --- a/libbeat/statestore/backend/memlog/memlog.go +++ b/libbeat/statestore/backend/memlog/memlog.go @@ -54,6 +54,9 @@ type Settings struct { // Checkpoint predicate that can trigger a registry file rotation. If not // configured, memlog will automatically trigger a checkpoint every 10MB. Checkpoint CheckpointPredicate + + // If set memlog will not check the version of the meta file. + IgnoreVersionCheck bool } // CheckpointPredicate is the type for configurable checkpoint checks. @@ -62,7 +65,7 @@ type CheckpointPredicate func(fileSize uint64) bool const defaultFileMode os.FileMode = 0600 -const defaultBufferSize = 4096 +const defaultBufferSize = 4 * 1024 func defaultCheckpoint(filesize uint64) bool { const limit = 10 * 1 << 20 // set rotation limit to 10MB by default @@ -110,7 +113,7 @@ func (r *Registry) Access(name string) (backend.Store, error) { home := filepath.Join(r.settings.Root, name) fileMode := r.settings.FileMode bufSz := r.settings.BufferSize - store, err := openStore(logger, home, fileMode, bufSz, r.settings.Checkpoint) + store, err := openStore(logger, home, fileMode, bufSz, r.settings.IgnoreVersionCheck, r.settings.Checkpoint) if err != nil { return nil, err } diff --git a/libbeat/statestore/backend/memlog/memlog_test.go b/libbeat/statestore/backend/memlog/memlog_test.go new file mode 100644 index 00000000000..d8fe048f69c --- /dev/null +++ b/libbeat/statestore/backend/memlog/memlog_test.go @@ -0,0 +1,258 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "encoding/json" + "io" + "io/ioutil" + "math" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/backend" + "github.com/elastic/beats/v7/libbeat/statestore/internal/storecompliance" +) + +func init() { + logp.DevelopmentSetup() +} + +func TestCompliance_Default(t *testing.T) { + storecompliance.TestBackendCompliance(t, func(testPath string) (backend.Registry, error) { + return New(logp.NewLogger("test"), Settings{Root: testPath}) + }) +} + +func TestCompliance_AlwaysCheckpoint(t *testing.T) { + storecompliance.TestBackendCompliance(t, func(testPath string) (backend.Registry, error) { + return New(logp.NewLogger("test"), Settings{ + Root: testPath, + Checkpoint: func(filesize uint64) bool { + return true + }, + }) + }) +} + +func TestLoadVersion1(t *testing.T) { + dataHome := "testdata/1" + + list, err := ioutil.ReadDir(dataHome) + if err != nil { + t.Fatal(err) + } + + cases := list[:0] + for _, info := range list { + if info.IsDir() { + cases = append(cases, info) + } + } + + for _, info := range cases { + name := filepath.Base(info.Name()) + t.Run(name, func(t *testing.T) { + testLoadVersion1Case(t, filepath.Join(dataHome, info.Name())) + }) + } +} + +func testLoadVersion1Case(t *testing.T, dataPath string) { + path, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create temporary test directory: %v", err) + } + defer os.RemoveAll(path) + + t.Logf("Test tmp dir: %v", path) + + if err := copyPath(path, dataPath); err != nil { + t.Fatalf("Failed to copy test file to the temporary directory: %v", err) + } + + // load expected test results + raw, err := ioutil.ReadFile(filepath.Join(path, "expected.json")) + if err != nil { + t.Fatalf("Failed to load expected.json: %v", err) + } + + expected := struct { + Txid uint64 + Datafile string + Entries map[string]interface{} + }{} + if err := json.Unmarshal(raw, &expected); err != nil { + t.Fatalf("Failed to parse expected.json: %v", err) + } + + // load store: + store, err := openStore(logp.NewLogger("test"), path, 0660, 4096, true, func(_ uint64) bool { + return false + }) + if err != nil { + t.Fatalf("Failed to load test store: %v", err) + } + defer store.Close() + + disk := store.disk + disk.removeOldDataFiles() + + // validate store: + assert.Equal(t, expected.Txid, disk.nextTxID-1) + if expected.Datafile != "" { + assert.Equal(t, filepath.Join(path, expected.Datafile), disk.activeDataFile.path) + } + + // check all keys in expected are known and do match stored values: + func() { + for key, val := range expected.Entries { + var tmp interface{} + err := store.Get(key, &tmp) + require.NoError(t, err, "error reading entry (key=%v)", key) + + assert.Equal(t, val, tmp, "failed when checking key '%s'", key) + } + }() + + // check store does not contain any additional keys + func() { + err = store.Each(func(key string, val statestore.ValueDecoder) (bool, error) { + _, exists := expected.Entries[string(key)] + if !exists { + t.Errorf("unexpected key: %s", key) + } + return true, nil + }) + assert.NoError(t, err) + }() +} + +func TestTxIDLessEqual(t *testing.T) { + cases := map[string]struct { + a, b uint64 + want bool + }{ + "is equal": {10, 10, true}, + "is less": {8, 9, true}, + "is bigger": {9, 8, false}, + "is less 0 with integer overflow": { + math.MaxUint64 - 2, 0, true, + }, + "is less random value with integer overflow": { + math.MaxUint64 - 2, 10, true, + }, + "is less with large ids": { + math.MaxUint64 - 10, math.MaxUint64 - 9, true, + }, + "is bigger with large ids": { + math.MaxUint64 - 9, math.MaxUint64 - 10, false, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + got := isTxIDLessEqual(test.a, test.b) + if got != test.want { + t.Fatalf("%v <= %v should be %v", test.a, test.b, test.want) + } + }) + } +} + +func copyPath(to, from string) error { + info, err := os.Stat(from) + if err != nil { + return err + } + + if info.IsDir() { + return copyDir(to, from) + } + if info.Mode().IsRegular() { + return copyFile(to, from) + } + + // ignore other file types + return nil +} + +func copyDir(to, from string) error { + if !isDir(to) { + info, err := os.Stat(from) + if err != nil { + return err + } + + if err := os.MkdirAll(to, info.Mode()); err != nil { + return err + } + } + + list, err := ioutil.ReadDir(from) + if err != nil { + return err + } + + for _, file := range list { + name := file.Name() + err := copyPath(filepath.Join(to, name), filepath.Join(from, name)) + if err != nil { + return err + } + } + return nil +} + +func copyFile(to, from string) error { + in, err := os.Open(from) + if err != nil { + return err + } + defer in.Close() + + info, err := in.Stat() + if err != nil { + return err + } + + out, err := os.OpenFile(to, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode()) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +func isFile(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} diff --git a/libbeat/statestore/backend/memlog/store.go b/libbeat/statestore/backend/memlog/store.go index 68bb6124a3a..29880a6aae8 100644 --- a/libbeat/statestore/backend/memlog/store.go +++ b/libbeat/statestore/backend/memlog/store.go @@ -18,9 +18,12 @@ package memlog import ( + "fmt" "os" + "path/filepath" "sync" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore/backend" @@ -56,51 +59,201 @@ type entry struct { // If an error in in the log file is detected, the store opening routine continues from the last known valid state and will trigger a checkpoint // operation on subsequent writes, also truncating the log file. // Old data files are scheduled for deletion later. -func openStore(log *logp.Logger, home string, mode os.FileMode, bufSz uint, checkpoint CheckpointPredicate) (*store, error) { - panic("TODO: implement me") +func openStore(log *logp.Logger, home string, mode os.FileMode, bufSz uint, ignoreVersionCheck bool, checkpoint CheckpointPredicate) (*store, error) { + fi, err := os.Stat(home) + if os.IsNotExist(err) { + err = os.MkdirAll(home, os.ModeDir|0770) + if err != nil { + return nil, err + } + + err = writeMetaFile(home, mode) + if err != nil { + return nil, err + } + } else if !fi.Mode().IsDir() { + return nil, fmt.Errorf("'%v' is not a directory", home) + } else { + if err := pathEnsurePermissions(filepath.Join(home, metaFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update meta file permissions: %w", err) + } + } + + if !ignoreVersionCheck { + meta, err := readMetaFile(home) + if err != nil { + return nil, err + } + if err := checkMeta(meta); err != nil { + return nil, err + } + } + + if err := pathEnsurePermissions(filepath.Join(home, activeDataFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update active file permissions: %w", err) + } + + dataFiles, err := listDataFiles(home) + if err != nil { + return nil, err + } + for _, df := range dataFiles { + if err := pathEnsurePermissions(df.path, mode); err != nil { + return nil, fmt.Errorf("failed to update data file permissions: %w", err) + } + } + if err := pathEnsurePermissions(filepath.Join(home, logFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update log file permissions: %w", err) + } + + tbl := map[string]entry{} + var txid uint64 + if L := len(dataFiles); L > 0 { + active := dataFiles[L-1] + txid = active.txid + if err := loadDataFile(active.path, tbl); err != nil { + return nil, err + } + } + + logp.Info("Loading data file of '%v' succeeded. Active transaction id=%v", home, txid) + + var entries uint + memstore := memstore{tbl} + txid, entries, err = loadLogFile(&memstore, txid, home) + logp.Info("Finished loading transaction log file for '%v'. Active transaction id=%v", home, txid) + + if err != nil { + // Error indicates the log file was incomplete or corrupted. + // Anyways, we already have the table in a valid state and will + // continue opening the store from here. + logp.Warn("Incomplete or corrupted log file in %v. Continue with last known complete and consistent state. Reason: %v", home, err) + } + + diskstore, err := newDiskStore(log, home, dataFiles, txid, mode, entries, err != nil, bufSz, checkpoint) + if err != nil { + return nil, err + } + + return &store{ + disk: diskstore, + mem: memstore, + }, nil } // Close closes access to the update log file and clears the in memory key // value store. Access to the store after close can lead to a panic. func (s *store) Close() error { - panic("TODO: implement me") + s.lock.Lock() + defer s.lock.Unlock() + s.mem = memstore{} + return s.disk.Close() } // Has checks if the key is known. The in memory store does not report any // errors. func (s *store) Has(key string) (bool, error) { - panic("TODO: implement me") + s.lock.RLock() + defer s.lock.RUnlock() + return s.mem.Has(key), nil } // Get retrieves and decodes the key-value pair into to. func (s *store) Get(key string, to interface{}) error { - panic("TODO: implement me") + s.lock.RLock() + defer s.lock.RUnlock() + + dec := s.mem.Get(key) + if dec == nil { + return errKeyUnknown + } + return dec.Decode(to) } // Set inserts or overwrites a key-value pair. // If encoding was successful the in-memory state will be updated and a // set-operation is logged to the diskstore. func (s *store) Set(key string, value interface{}) error { - panic("TODO: implement me") + var tmp common.MapStr + if err := typeconv.Convert(&tmp, value); err != nil { + return err + } + + s.lock.Lock() + defer s.lock.Unlock() + + s.mem.Set(key, tmp) + return s.logOperation(&opSet{K: key, V: tmp}) } // Remove removes a key from the in memory store and logs a remove operation to // the diskstore. The operation does not check if the key exists. func (s *store) Remove(key string) error { - panic("TODO: implement me") + s.lock.Lock() + defer s.lock.Unlock() + + s.mem.Remove(key) + return s.logOperation(&opRemove{K: key}) +} + +// lopOperation ensures that the diskstore reflects the recent changes to the +// in memory store by either triggering a checkpoint operations or adding the +// operation type to the update log file. +func (s *store) logOperation(op op) error { + if s.disk.mustCheckpoint() { + err := s.disk.WriteCheckpoint(s.mem.table) + if err != nil { + // if writing the new checkpoint file failed we try to fallback to + // appending the log operation. + // TODO: make append configurable and retry checkpointing with backoff. + s.disk.LogOperation(op) + } + + return err + } + + return s.disk.LogOperation(op) } // Each iterates over all key-value pairs in the store. func (s *store) Each(fn func(string, backend.ValueDecoder) (bool, error)) error { - panic("TODO: implement me") + s.lock.RLock() + defer s.lock.RUnlock() + + for k, entry := range s.mem.table { + cont, err := fn(k, entry) + if !cont || err != nil { + return err + } + } + + return nil +} + +func (m *memstore) Has(key string) bool { + _, exists := m.table[key] + return exists +} + +func (m *memstore) Get(key string) backend.ValueDecoder { + entry, exists := m.table[key] + if !exists { + return nil + } + return entry } -func (s *memstore) Set(key string, value interface{}) error { - panic("TODO: implement me") +func (m *memstore) Set(key string, value common.MapStr) { + m.table[key] = entry{value: value} } -func (s *memstore) Remove(key string) error { - panic("TODO: implement me") +func (m *memstore) Remove(key string) bool { + _, exists := m.table[key] + if !exists { + return false + } + delete(m.table, key) + return true } func (e entry) Decode(to interface{}) error { diff --git a/libbeat/statestore/backend/memlog/testdata/1/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/Readme.md new file mode 100644 index 00000000000..7532ab9b942 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/Readme.md @@ -0,0 +1 @@ +Sample disk stores version 1 diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json new file mode 100644 index 00000000000..90bd45d5de1 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md new file mode 100644 index 00000000000..9c7b4584885 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md @@ -0,0 +1,2 @@ +Store file with transaction ID and log with with transaction id 3. +The data file 2.json is missing, which leads to all entries in the log file to be ignored. diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json new file mode 100644 index 00000000000..f691496a5b2 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json new file mode 100644 index 00000000000..5b5d8f4e0f6 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json @@ -0,0 +1,6 @@ +{"op": "set", "id": 3} +{"K":"key3","V":{"a":3}} +{"op": "set", "id": 3} +{"K":"key3","V":{"a":3}} +{"op": "set", "id": 3} +{"K":"key5","V":{"a":5}} diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json new file mode 100644 index 00000000000..bcb050a2821 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/1.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/1.json new file mode 100644 index 00000000000..90bd45d5de1 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/Readme.md new file mode 100644 index 00000000000..5272ae569bd --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/Readme.md @@ -0,0 +1 @@ +Store with valid data file and valid log file with logged updates. diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/expected.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/expected.json new file mode 100644 index 00000000000..c7a3023283c --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 4, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/log.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/log.json new file mode 100644 index 00000000000..c7edd4f783b --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/log.json @@ -0,0 +1,6 @@ +{"op":"set", "id": 2} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 3} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 4} +{"K":"key5","V":{"a":5}} diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/meta.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/meta.json new file mode 100644 index 00000000000..bcb050a2821 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/1.json b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/1.json new file mode 100644 index 00000000000..0460d15de49 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/1.json @@ -0,0 +1,8 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2}, +{"_key":"key3","a":3}, +{"_key":"key4","a":4}, +{"_key":"key5","a":5} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/Readme.md new file mode 100644 index 00000000000..d5cc1f55440 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/Readme.md @@ -0,0 +1,2 @@ +Valid store without log.json. All entries are read from 1.json only. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/expected.json b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/expected.json new file mode 100644 index 00000000000..8c643ce7fda --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/meta.json b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/meta.json new file mode 100644 index 00000000000..bcb050a2821 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/1.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/1.json new file mode 100644 index 00000000000..90bd45d5de1 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md new file mode 100644 index 00000000000..a3df15b9283 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md @@ -0,0 +1,2 @@ +Valid data file with incomplete log.json. The last entry in the log file has +the data part missing, which will fail the log file to be read. diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/expected.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/expected.json new file mode 100644 index 00000000000..f691496a5b2 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/log.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/log.json new file mode 100644 index 00000000000..50462314cb6 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/log.json @@ -0,0 +1 @@ +{"op":"set", "id": 2} diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/meta.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/meta.json new file mode 100644 index 00000000000..bcb050a2821 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md new file mode 100644 index 00000000000..0c8ac8f51ab --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md @@ -0,0 +1,3 @@ +Checkpoint file does not exist yet. All entries are read from the log file. +Contents is missing with the last operation in the log file triggereing a parse error. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json new file mode 100644 index 00000000000..356fb55782d --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 6, + "datafile": "", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json new file mode 100644 index 00000000000..a0cc0a36eb9 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json @@ -0,0 +1,13 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"set", "id": 4} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 5} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 6} +{"K":"key5","V":{"a":5}} +{"op":"set", diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json new file mode 100644 index 00000000000..cd576d0b504 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json @@ -0,0 +1 @@ +{"version":"1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/Readme.md new file mode 100644 index 00000000000..725042043f3 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/Readme.md @@ -0,0 +1,2 @@ +Checkpoint file does not exist yet. All entries are read from the log file. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/expected.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/expected.json new file mode 100644 index 00000000000..356fb55782d --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 6, + "datafile": "", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/log.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/log.json new file mode 100644 index 00000000000..f00679f2255 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/log.json @@ -0,0 +1,12 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"set", "id": 4} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 5} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 6} +{"K":"key5","V":{"a":5}} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/meta.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/meta.json new file mode 100644 index 00000000000..cd576d0b504 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/meta.json @@ -0,0 +1 @@ +{"version":"1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/1.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/1.json new file mode 100644 index 00000000000..0460d15de49 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/1.json @@ -0,0 +1,8 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2}, +{"_key":"key3","a":3}, +{"_key":"key4","a":4}, +{"_key":"key5","a":5} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/2.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/2.json new file mode 100644 index 00000000000..c3d3f12f859 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/2.json @@ -0,0 +1,3 @@ +[ +{"_key":"key0","a":0} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/3.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/3.json new file mode 100644 index 00000000000..9ac6abf9862 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/3.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":30}, +{"_key":"key1","a":31}, +{"_key":"key2","a":32} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md new file mode 100644 index 00000000000..cdea1159819 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md @@ -0,0 +1,2 @@ +Store with old data files that failed to be cleaned up. Only entries from data file 3.json should be loaded. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/expected.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/expected.json new file mode 100644 index 00000000000..f671203914a --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 3, + "datafile": "3.json", + "entries": { + "key0": {"a": 30}, + "key1": {"a": 31}, + "key2": {"a": 32} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/meta.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/meta.json new file mode 100644 index 00000000000..bcb050a2821 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json new file mode 100644 index 00000000000..90bd45d5de1 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md new file mode 100644 index 00000000000..9d99a9ae95a --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md @@ -0,0 +1,3 @@ +Due to restart the log file was not truncated after the last checkpoint was +written. Update with ID 0 adds the removed key0 again to the store. All entries +in log.json should be ignored. diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json new file mode 100644 index 00000000000..4e9547000e7 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 5, + "datafile": "5.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json new file mode 100644 index 00000000000..426ce6a4f61 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json @@ -0,0 +1,8 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"remove", "id": 4} +{"K":"key0"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json new file mode 100644 index 00000000000..bcb050a2821 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/util.go b/libbeat/statestore/backend/memlog/util.go index f02311392b9..2027c87adca 100644 --- a/libbeat/statestore/backend/memlog/util.go +++ b/libbeat/statestore/backend/memlog/util.go @@ -20,6 +20,7 @@ package memlog import ( "io" "os" + "runtime" "syscall" ) @@ -76,3 +77,45 @@ func trySyncPath(path string) { defer f.Close() syncFile(f) } + +// pathEnsurePermissions checks if the file permissions for the given file match wantPerm. +// The permissions are updated using chmod if needed. +// No file will be created if the file does not yet exist. +func pathEnsurePermissions(path string, wantPerm os.FileMode) error { + f, err := os.OpenFile(path, os.O_RDWR, wantPerm) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + defer f.Close() + return fileEnsurePermissions(f, wantPerm) +} + +// fileEnsurePermissions checks if the file permissions for the given file +// matches wantPerm. If not fileEnsurePermissions tries to update +// the current permissions via chmod. +// The file is not created or updated if it does not exist. +func fileEnsurePermissions(f *os.File, wantPerm os.FileMode) error { + if runtime.GOOS == "windows" { + return nil + } + + fi, err := f.Stat() + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + wantPerm = wantPerm & os.ModePerm + perm := fi.Mode() & os.ModePerm + if wantPerm == perm { + return nil + } + + return f.Chmod((fi.Mode() &^ os.ModePerm) | wantPerm) +} diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 52eba34a1ff..37ec6f150e0 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -133,7 +133,6 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c moduleConfig := common.MapStr{ "module": mod, "metricsets": msets, - "hosts": hosts, "timeout": tout, "period": ival, "enabled": true, @@ -154,15 +153,30 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c moduleConfig["password"] = password } - logp.Debug("hints.builder", "generated config: %v", moduleConfig) + // If there are hosts that match, ensure that there is a module config for each valid host. + // We do this because every config that is from a Pod that has an exposed port will generate a valid + // module config. However, the pod level hint will generate a config with all hosts that are defined in the + // config. To make sure that these pod level configs get deduped, it is essential that we generate exactly one + // module config per host. + if len(hosts) != 0 { + for _, h := range hosts { + mod := moduleConfig.Clone() + mod["hosts"] = []string{h} + + logp.Debug("hints.builder", "generated config: %v", mod) + + // Create config object + cfg := m.generateConfig(mod) + configs = append(configs, cfg) + } + } else { + logp.Debug("hints.builder", "generated config: %v", moduleConfig) - // Create config object - cfg, err := common.NewConfigFrom(moduleConfig) - if err != nil { - logp.Debug("hints.builder", "config merge failed with error: %v", err) + // Create config object + cfg := m.generateConfig(moduleConfig) + configs = append(configs, cfg) } - logp.Debug("hints.builder", "generated config: %+v", common.DebugString(cfg, true)) - configs = append(configs, cfg) + } // Apply information in event to the template to generate the final config @@ -171,6 +185,15 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c return template.ApplyConfigTemplate(event, configs, options...) } +func (m *metricHints) generateConfig(mod common.MapStr) *common.Config { + cfg, err := common.NewConfigFrom(mod) + if err != nil { + logp.Debug("hints.builder", "config merge failed with error: %v", err) + } + logp.Debug("hints.builder", "generated config: %+v", common.DebugString(cfg, true)) + return cfg +} + func (m *metricHints) getModule(hints common.MapStr) string { return builder.GetHintString(hints, m.Key, module) } diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index d2d0462a047..4badd9c2a02 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -409,6 +409,66 @@ func TestGenerateHints(t *testing.T) { }, }, }, + { + message: "Module with multiple hosts returns the right number of hints. Pod level hints need to be one per host", + event: bus.Event{ + "host": "1.2.3.4", + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "mockmoduledefaults", + "namespace": "test", + "hosts": "${data.host}:9090, ${data.host}:9091", + }, + }, + }, + len: 2, + result: []common.MapStr{ + { + "module": "mockmoduledefaults", + "namespace": "test", + "metricsets": []string{"default"}, + "timeout": "3s", + "period": "1m", + "enabled": true, + "hosts": []interface{}{"1.2.3.4:9090"}, + }, + { + "module": "mockmoduledefaults", + "namespace": "test", + "metricsets": []string{"default"}, + "timeout": "3s", + "period": "1m", + "enabled": true, + "hosts": []interface{}{"1.2.3.4:9091"}, + }, + }, + }, + { + message: "Module with multiple hosts and an exposed port creates a config for just the exposed port", + event: bus.Event{ + "host": "1.2.3.4", + "port": 9091, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "mockmoduledefaults", + "namespace": "test", + "hosts": "${data.host}:9090, ${data.host}:9091", + }, + }, + }, + len: 1, + result: []common.MapStr{ + { + "module": "mockmoduledefaults", + "namespace": "test", + "metricsets": []string{"default"}, + "timeout": "3s", + "period": "1m", + "enabled": true, + "hosts": []interface{}{"1.2.3.4:9091"}, + }, + }, + }, } for _, test := range tests { mockRegister := mb.NewRegister() @@ -423,7 +483,7 @@ func TestGenerateHints(t *testing.T) { logger: logp.NewLogger("hints.builder"), } cfgs := m.CreateConfig(test.event) - assert.Equal(t, len(cfgs), test.len) + assert.Equal(t, len(cfgs), test.len, test.message) // The check below helps skipping config validation if there is no config supposed to be emitted. if len(cfgs) == 0 { diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index fb06fcfb57d..2e6f35dfece 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19417,6 +19417,7 @@ type: long *`haproxy.info.compress.bps.in`*:: + -- +Incoming compressed data in bits per second. type: long @@ -19426,6 +19427,7 @@ type: long *`haproxy.info.compress.bps.out`*:: + -- +Outgoing compressed data in bits per second. type: long @@ -19435,6 +19437,7 @@ type: long *`haproxy.info.compress.bps.rate_limit`*:: + -- +Rate limit of compressed data in bits per second. type: long @@ -19456,6 +19459,7 @@ type: long *`haproxy.info.connection.rate.value`*:: + -- +Number of connections in the last second. type: long @@ -19465,6 +19469,7 @@ type: long *`haproxy.info.connection.rate.limit`*:: + -- +Rate limit of connections. type: long @@ -19474,6 +19479,7 @@ type: long *`haproxy.info.connection.rate.max`*:: + -- +Maximum rate of connections. type: long @@ -19552,6 +19558,7 @@ type: long *`haproxy.info.requests.total`*:: + -- +Total number of requests. type: long @@ -19561,6 +19568,7 @@ type: long *`haproxy.info.sockets.max`*:: + -- +Maximum number of sockets. type: long @@ -19570,6 +19578,7 @@ type: long *`haproxy.info.requests.max`*:: + -- +Maximum number of requests. type: long @@ -19585,6 +19594,7 @@ type: long *`haproxy.info.pipes.used`*:: + -- +Number of used pipes during kernel-based tcp splicing. type: integer @@ -19594,6 +19604,7 @@ type: integer *`haproxy.info.pipes.free`*:: + -- +Number of free pipes. type: integer @@ -19603,6 +19614,7 @@ type: integer *`haproxy.info.pipes.max`*:: + -- +Maximum number of used pipes. type: integer @@ -19618,6 +19630,7 @@ None *`haproxy.info.session.rate.value`*:: + -- +Rate of session per seconds. type: integer @@ -19627,6 +19640,7 @@ type: integer *`haproxy.info.session.rate.limit`*:: + -- +Rate limit of sessions. type: integer @@ -19636,6 +19650,7 @@ type: integer *`haproxy.info.session.rate.max`*:: + -- +Maximum rate of sessions. type: integer @@ -19651,7 +19666,8 @@ None *`haproxy.info.ssl.rate.value`*:: + -- -None +Rate of SSL requests. + type: integer @@ -19660,7 +19676,8 @@ type: integer *`haproxy.info.ssl.rate.limit`*:: + -- -None +Rate limit of SSL requests. + type: integer @@ -19669,7 +19686,8 @@ type: integer *`haproxy.info.ssl.rate.max`*:: + -- -None +Maximum rate of SSL requests. + type: integer @@ -19684,7 +19702,8 @@ None *`haproxy.info.ssl.frontend.key_rate.value`*:: + -- -None +Key rate of SSL frontend. + type: integer @@ -19693,7 +19712,8 @@ type: integer *`haproxy.info.ssl.frontend.key_rate.max`*:: + -- -None +Maximum key rate of SSL frontend. + type: integer @@ -19702,7 +19722,8 @@ type: integer *`haproxy.info.ssl.frontend.session_reuse.pct`*:: + -- -None +Rate of reuse of SSL frontend sessions. + type: scaled_float @@ -19719,7 +19740,8 @@ None *`haproxy.info.ssl.backend.key_rate.value`*:: + -- -None +Key rate of SSL backend sessions. + type: integer @@ -19728,7 +19750,8 @@ type: integer *`haproxy.info.ssl.backend.key_rate.max`*:: + -- -MaxConnRate +Maximum key rate of SSL backend sessions. + type: integer @@ -19737,7 +19760,8 @@ type: integer *`haproxy.info.ssl.cached_lookups`*:: + -- -None +Number of SSL cache lookups. + type: long @@ -19746,7 +19770,8 @@ type: long *`haproxy.info.ssl.cache_misses`*:: + -- -None +Number of SSL cache misses. + type: long @@ -19761,6 +19786,7 @@ type: long *`haproxy.info.zlib_mem_usage.value`*:: + -- +Memory usage of zlib. type: integer @@ -19770,6 +19796,7 @@ type: integer *`haproxy.info.zlib_mem_usage.max`*:: + -- +Maximum memory usage of zlib. type: integer @@ -19779,6 +19806,7 @@ type: integer *`haproxy.info.idle.pct`*:: + -- +Percentage of idle time. type: scaled_float diff --git a/metricbeat/module/haproxy/fields.go b/metricbeat/module/haproxy/fields.go index 378af3d10eb..2cf687fbb9d 100644 --- a/metricbeat/module/haproxy/fields.go +++ b/metricbeat/module/haproxy/fields.go @@ -32,5 +32,5 @@ func init() { // AssetHaproxy returns asset data. // This is the base64 encoded gzipped contents of module/haproxy. func AssetHaproxy() string { - return "eJzsXW9z2zaTf59PseM3lZ+T1aR1m5nMtDON8/SSqeN4bOeeFzc3KkSuRJxJgAVAy+qnv8EfUhQFkpBF2pm5h28SS+TubxfA/sOCOoN73LyDhOSCP25eASiqUnwHJx9/u9afnLwCiFFGguaKcvYOfn0FAOC+hc88LlJ8BSATLtQ84mxJV+9gSVKpPxWYIpH4DlZE34NKUbaS7+C/T6RMT6ZwkiiVn/zPK4AlxTSW7wzxM2AkwzoofalNrgkJXuTuEw+uOrYMlaCRnLkv6hzqXChb8upDH5sOVvr6T2QoSGroiIzoW4AseKEqILngEUqJFRR97aqmvJog60ArMjvflohTzlaNLzpA6+uqyBYogC99ALsQzFmRDYTh2lIEZrD0sFeJQBIPL76j2yc8jb2cSUpJE1NOVFKpa7b/ZEZXglhgShT4NJ19+tCDWBRs/leBe/QP1ZiXuFQ8z+keieNHoyQM/8sXfWOibxkcAEnTEN4FM0jJIsX5KDhqDELwpFQqbYqGB1JR7kEQC57nGM9TvhoehCMOmngPjkUhN/Ocp+kY01MTB0e8B8eS0BTjuUDJ00JTHl4rlgXUWPTZUCLvj4XhXw65ohnOJEYDCXlRCIFMOcJAGUiMOOu10xlmXGxmGXmcLTYq3Fta7/0OfA/1QP1MHmlWZEAyXjClx8WCgEKSlYFuiMJEJQjffcYsI4/zz++/gweSFggRZw8oFMaguL3z1COjf6q3StiMYMATXewJUpLlhdr7rotwEPE6A8UVSb13dIxQefWMRnnV1qzRvtSziReqZf40IQqimo5zaIS/PaDQE8Ti44XKC2X4hg5/jm32/qjhJ5GiDz7hOwUPELrmZA0LK0DHgJSIIs4YRgqbgdSwoCourbj8o8B5cy4PMQhpyiMyjsi39G+0kS9u+Rg5AsaikCMPQ4b6f9IwgqXg2dNwWt84KlLnft280e5Xm3CNrxQidA4VKc2omrOBnGfpkVgFlefIYElTlNrTGY2W+Uk3sohnuUAZbmZCogZ/pltnu8ibHLu5hnPu4g47hYGBzX8nN7/LHY2d9jJzM+fG5Now35T75/d4s6jFiT/nNDIx3nMO7eij2mSYkcex2VUTySYFIxj1Mt2o2fIAH9MWxx6L5k7TPRCLlOlsfP3c3l4+Ade4enoaJv+0PRZR6XkPxzQunsOwJETE8+EAeZ2CwL8KlEp6J8cwNUoe3aOSnpEehHwlwEj0c5ofkNx3zoU+P9kZ01OmcIXiCE9ZheQC293xcGy6Ju5xXKqZhVI+NZp5QggzawsihtacYdYWQIzCbJyx8g+aDE/Tv4UBe8bherbBCrERnClk7eYoJGx/Yqx+j5t55/CFiBsuspd1XzQ9NGNnyuYCC4mzPOpOHWREUozny5STthvLQnqOIvLHouFAq2oAie7/PScGYKzjwQvO2I0vMa6yLBIlZh+P3xcdZZiAGLCbxTyjnn6GpzHoNv1/p3QxzzCbmx2ZY2OqAx3Ccznv8cMeGqc+E9FrGrpNQguCbafBDr2DG3RuFVESIp6mtrZvKsoD9Ob4YwxFVOEP2u9xs+aiacN6UrhbQw8mX6+n8OHLv66mcPXl8v0UPv/26epuClzY/00eKDmdzWa+jcM6vDXSVeIfvsOLzDb5tiRhsuSiNNPy1CCTKB5Q7NxgP/Lub9ZhxnzNFM2ObVzZBVoShcl2L/l0Br/XcE9BJVS6bVkqTZW8BQtUZfR1wlMsSUyBcWU+lkVWbrFUnMtHnBoCKu+cIVNzLbhXF/5l3bezXtI1RGDy+pcy5prCm18qQX74xcI0Y/njLzaZ/r5sBukbwrJX7NvrmoLJazMQSyqkAsqkIizCKbwBO0P1xJgCYTFIDpz1CaqVRCOc678GXPeWquEBk99vvlzd/fPqg8VdDdb73y7+KD+tho0LIGxjH9wuueBxo+zZ2ibem21vynoQ8UI9M6T2JoGqxk6kmkcJYS3BxJPWZa0HzlonkJRFaCyGZghfr89+1U5Aj7H+9+zXr9egBGGSapr97ZNcqWd04OVVlo5LACUVsmpaRFgnyECmfC0VEfupA5WuccBMc8ZLS73cPqPvocze1btybUQwSAGyvKyv2e67arsvgVQCElnxnQJSlaAwSmC43qPlUjNppDWqEXgWU5kTFSWUrazzcs7E+S4TpYDAnAtlHNgeVa3uJr76ENQQ9s0ooY1OPGux8Ycr79OH0mWazu/vHSS6tKwoW+nhRUYWaS+4iPN7OqBFvjD03IxzKB0+N4eNyXXfWO7lX8449xkVTuL5gqSERZSt5iRdcUFV4u+1fpIMl5zEUHGAikOvKg/evg1ucxlrZ+iiyIqUmCYj1mzwCdwEEagEHbmDpDQPW2Sg+K65cDBCdiNphjPy4AN1LOKyVc3hhLInM5PANdDKR715/cP5dmckQMldO/QDNnWZ5pxdqNbLBmIccDesfn2kqwSl2pqVLdxZSydgEx1RCrNcjbbH6lMloFRkkVKZZNqzOwhhi6qQ+LxYDcsQbDROD+8W+Yb7XrU8O/OfPBCamgMFOuSwQxEGfIymkuZ1qXkAZ8fi3qkxjjuetkTaOp1hlGE1cjnWXdO6DjShyldnHROgZtl3Qsn6ieGjihjZOJ77xnk2xwEWGJFCokvaCkHVRs/cCEVniGGvf5j4/e7i2obuVNbJEch0mI8xqCg/c5rStJW2uKJIcdZJ9uPdXQ/dRKktYR3DEpHTJukeFc8Xm/l2pc71s0fV8w9Xu1ZfTTuV0ddIDhKj2gh7WRkcjGABzAm/+Bk6vyyjbWznq5zuZZ8u+PYS5suKWEl8TVXCC7UNgImUdMWCol+niHHbqKo4bVftAfBQCL53WGMIaG5eOQYzuOXbNDTnUlLtN81Uk0AE9ng2bT6QiHQDCkVGmT3QXHXhRylFpqawwCUXtjRVTtyEaDuDzBy0aaUtkMQWapNo6yP2a5NytHdJV7dFKZc7PfmtDzwQQXkhYUG2s7oJqt3GlmLrFM1azvaCCez4vLKC80ypZR1onblJNBk3FSi72syS9lLtqTH11JP8JNfUwiBrsjFqDz8HpLOj0dbTVoPVpFAJUYAs4gVTKDRm5uawEhvKVqC4l1SVLW9T+kCzaXePvETN14ooEwuTNC2JbQcpTwtpNgHqCZtRFzDuX0BESh5Rc7xG22AgkBOhaFSkpNrIm8giSoDIekkLEvKgFcD8CnDHvHq2/WCkjv3t9VKN+s1rO7dMeFZNsBzLnadGjQJTkmtz1qxV+AQYvg+/ee0fK2oT48V759sgCoyQPgRZaqoXe4T52McOa3y6KmctGZTMOZPhLS3BKdSz2FcLvgpd7mq77yxKixib7iUmivitoiBMLlFIIAtuDlMvNnU3NHFvk5lp0zlzttjdeupfVzaQMv7RWc/+8OkfsBZUlRIBZ7VYwvWjw2TN2XcKFgjWocTNXTAN8bSF/JLQtBAIJM9T43qWNFVabMVdNNaYEAELcfyycTXSYXXjcov+7uI6xGf0ZPsH9z7VrxuHPCzj38u8vTT7s3Gd/JldvxNu0EGUYHRv6ionIcc3lPK5yec89vbmcWxn5Ox6OTwmbHnz+AgRj0Ormj+8CMgfDgP544uA/PEwkOcvAvL8MJA/vQjInw4DaRzOC8C0jk4DlTDJBVc84qn1Y8GvI0mQxHvgB4hFBBqverBJO6iW3/q+gH4+wbwgcC8IQgYbwgccfK8tsKNVKhfWRDDKvG9X8g72oWeggkd7vFJmbYvCFTXLFpsXP5/Jas1fwZC6j/4cUfI173EshEk8bTnasHrBrG73/WwH6OhJBYVh6gUHHMsIKRmUQh9eLeiUo39rdxA5anMqLTd6Ga59UgUCH/BUzCElj3DQXrNpIvlgoznMCRTvqQTo6SULVJI7oeA2HMw0TJCkKrGSzuAL05lOr2P8evWHxfwrFOye8XVb4f7T1afyRsqooiSlf++/6a+C9+Xij3/e3Oi7XfptQpqWuy/Pv/zhaBv0kBOpF5Q2gGSDAs6nwDgUuR5384kEhVLpRNy1JbZSvvvy9c5QtpTenJ33bGlcnl98uYLGI7WSbi74IsVsalJlfCRZ7m1f2b1OLuqNMctCYnwCExXlIKQ6NSnnFQfBC4WgOCRcqhOY0CjL/RUJgMufe3T2c+uDDZX8DJPb28vTPrX8fHN7DTuPUfZAUhpvywxnsBvBtpF62wP9bceDF/UHtQUwTdgkTTf7ZHbGCM5fn5uYu3ewYir1nDrj7Oz89XkrloYa38JEB/jf336+u+5V5tuGMt8eoczbu9tdUrsFvl0lmByknpG1x4M8HqNF8NKA/OnsrWEwBbrc9hmFFJ0KexhlBGR32yKZ2XmiChTn93o9LimjMmkxtf2g7e0z/eg43uDOliCLVLV7hGCYOk8ZNZQs39+mYYXEkmSFTHVrb8CXZYz//joruJ1kThfrhKbYPA9Q5CELwu+yh0NbHX/ZHnnxd6HUz+95ada2q7ekygr6znk+WKD27Vq2qU4lEhNuE39s0jj2V6dtLB+JknIXuCdGtL0HbnvCGyweecIoJoq075k0ex/6CgFapGNDWnsFFwn2zpz0qSZAPeAO3qEwxwQLRv8yO1GSxgjEnkkJ2Y/wj9tAAEPHsGw3qO+p+bcj3FZbrF3Ozi5Z6b5dr0WI5N1vjh1I8EbDg7VfRGB1OCxDwkxbhv5CJbjR33qJWk+zMcc9I8LK7WFPP1C6PTvTun6h8coKb23heVRh2XecNPYbHveyz70gb+wMteXYKfQ5kK6znh3o6tdeT4aLWe1bmZe2Ucqe6SqVEzD8badWIWAGjC4UZqZbrTL3hwi22Njs5hsYLyuMmfIlKiOPEXkrFEwurr9+//5ftu4U1IRUGr6Xl7GxVWOEXaPYDlrvQcj6j+js4h9xOY/k/L5al2xEav8BlDoSz7F8GCbVsS8Q0fRDNg+6ctgBYGj6powzNZv/U+fYpmUtLXgLr/0HY8abLQO/cK5+1arORjJXe55k5NH8fdrovXEHy1VizhovWzOIqg+8JKTTjdcwKSMPxsOtTW//zVEa0Bk5cW04FumTz256J4zJiZ95wvy7nj1iPfv/bR17Www2c9r1Y2MMJ0V+0vpQVfRsPrQk1PzGnVQ8PzGAYr5mJwGFlO0kHWeKfywyk+qQ2Jyp1AmCViNfugKXXV5B7fgdXu0oq/Vfxr5WylxsHLSA2I3udcMOhOmGytr7H0LxLEnaXsQ8Cs/vJE0Px2ON2TiIPlpDmRNBMlRuu97ggveo1ogMXptU+0/7qR6rP//D/aEV9efZmyFK70cJ8cFR33EB1noapxmyLjx7vDBst1LLNA/RQaAeYG/Kd9YumwBb5v3AAHfXwEEAWxfCwBB9i6IsznlXhY0zdtaG/ah/hUCwG4EAV3KIkK0upW3PxxtMSl6IaITDDCSOPb9hsyWu8NEXvYR0uFvK1RuBjABd4r76vwAAAP//Au56Dg==" + return "eJzsXe1z2zaT/56/Ysdfqjwnq0nrNjOZaWca5+klk8TxxM49H25uVIhcmTiDAAuAltW//gYvpCgKFCGLtHPTR1/a6GXx28Vi37BLn8Itrl9DRgop7tfPADTVDF/DybvfLs07J88AUlSJpIWmgr+GX58BAPhP4ZNIS4bPAFQmpJ4ngi/pzWtYEqbMuxIZEoWv4YaY76DWlN+o1/DfJ0qxkymcZFoXJ//zDGBJkaXqtSV+Cpzk2ARlXnpdGEJSlIV/J4CriS1HLWmiZv6D5grNVShfivrN0DJ7ljKv/0SOkjBLR+bEfAXIQpS6BlJIkaBSWEMxr23RVK82yCbQmszWpxViJvhN64M9oM3roswXKEEsQwD3IZjzMh8Iw6WjCNxi6VleZxJJOjz7nm4f8zQNrkwYJW1MBdFZLa7Z7i9zeiOJA6ZliQ+T2fu3PYhlyed/lrhD/1CJBYkrLYqC7pA4fjcqwvC/YtG3J+YrgwMgjMWsXXKLlCwYzkfB0VggBg+jShtTNDyQmnIPglSKosB0zsTN8CA8cTDEe3AsSrWeF4KxMdTTEAdPvAfHklCG6VyiEqw0lIeXilsCGkv02VCibo+FET4OhaY5zhQmAzF5XkqJXHvCQDkoTATvtdM55kKuZzm5ny3WOt5bOu/9GkI/6oH6idzTvMyB5KLk2uyLAwGlIjcWuiUKE50hfPcJ85zczz+9+Q7uCCsREsHvUGpMQQv3zecBHsOq3slhO4KBQHSxw0hFVpR657N9hKOINxfQQhMW/MaeHapePbtRvRpn1kpfGW0Spe7QnzZESXTbcQ6N8Lc7lEZBHD5R6qLUdt3Y7S+wy94ftf0k0fQuxPxexiOYbjhZu4RjYM+GVIgSwTkmGtuB1LCg6lU6cYV3QYi2Lg+xCYyJhIzD8hX9C13ki5t1LB8Re1GqkbchR/N/yi4ESynyh+F0vnFUpN79er0x7teYcIOvYiJWh0pGc6rnfCDnWXkkXkMVBXJYUobKeDor0So/2Y8sEXkhUcWbmZioIZzpNpddFO0V968av/K+1WGrMDCy+X/PE5GbTKcSMaaQEk1swEC1ggKlD3oifVbYbw+J+XOpb8SgmI2/m1vtHxn6F6IR7ELO1j8Qf8slURE+s+OdjI7A5DGPho1bHy18a5pXyq3pYkTpw9TsKTSshh2JMSf3IyOs/ILRoXiMtcq7lGwEl1ole4fh6coijkVzbegeiEUpNhtfPldXHx+Aa1w5PQxTWNePRVTp9+GYxsVzGJaMyHQ+HKCg+5L4Z4lKq6ByPDTsdAqxCTrrRfZjUSK5Ra0COjFc/FutESmVMaFESqWgxQG1nb0o+kKKvSkd5RpvUB6bK9lszvIEaSlNCHmLkiM7XRDziU4KUAWjSXddtYl4KbE7CBoIsVnDIT7SfhyFZ1d/NpLsO1eo1EPj0wcEpbOusHAAKXzxAYtnqRGmx2yORdcVAQ6Frg78PMZoYKOrjtwWXnRRQqn4utb/A90xQUGP7d2B9WhK8xBwj6Y4B4JbSsE18m6XEpOlPjA1vcX1fK9CQYSI2lj2ZFQfcL0lpor3yJSvhtuX+w0CttrX2+NAezsyl1gqnBXJ/rxaJYRhOl8yQbq+WN29FSiTcAJ1AJPVcbfo2hweYpwXJLn9m2qxZz1GWkHYT6rN3eDbeBOSZLZhQdyWe+rNQ1wV2IzULAd+tR5U85wGer3GwuQWi40L/mJ0Mc8xn9v77WNTlAOjhREDhU/NW3uxtHx+C6F/Ho+rvrxIWcgy91rk/Za4rzPN/cijNBDANoU8a6NTemvhg3surzTRChLBmLuutZeEA7RbhqNgTXQZTsRvcb0Ssu0heqR0ZenB5OvlFN5+/tfFFC4+f3wzhU+/vb+4noKQ7v8md5Q8n81moV6QJrwV0pssvM8PLeA4kjBZClkZU/XcIlMo71BufcG9FWxZacJMxYobZRgUaEUUJpv2oOcz+L2Bewo6o8p32lBlbw86sEB9M7rKBMOKxBS40PZtVebVrXm9cvUTL4aIy1TBkeu5YTwoi7DF6GuWquhaIjB58UsV8Ezh5S81Iz/84mDavfzxF1cT+77q7+vbwqr999trhIXJC7sRSyqVBsqVJjzBKbwEp6FGMaZATEwgQPA+Ro2QaIJz868Bz72jateAye9fPl9c//PircNdb9ab384/VO/W2yYkEL52P9wcueh9o/zROuHe2E4mynsQiVI/MqTuvq/6gpAoPU8ywjsimgedy0Zbs7NOoChPcHOH+fXy9FfjBMwem/+e/vr1ErQkXFFDs78jXmj9BJ6+uo+qAFRUjOvftoiwypCDYmKlNJG7qR1VvhfMqjkXlaVebn5jvkO5+1bvyXURwai3GsbuKyA1g0TV604Bqc5QWiFwXO3QqnISy60VjcTTlKqC6CSj/MY5L+9MvO+yUQpILITU1oHtUDXibuNrbkEDYZ9GSWN00lmHjT9ceO/fVi7TDvN87yHRpVuK8huzvcjJgvWCS4S4pQNa5HNLz2ucR+nxeR22Jtd/4lav/uWNc59RESSdLwgjPKH8Zk7YjZBUZ+HxmQfx8FGQFOoVoF5h+O6V6M7Fsa6bz8u8ZMT2jfJQc0hM2RS1pCM3BVbmYYMMtNg2Fx5GTIuDyZ/IXQjUsYir7mOPE6o2+1yBMEBrH/XyxQ9nh9amR5VwsyFoG2pvS9BW19k4d/7v6E2GSm/MygburKO5u42OaI15oUdr3AiJElBpsmBUZbnx7B5C3KEqFT4uVrtkDDaassOb5b7hUQZbVGnqP7kjlNkZMRNyuK2IA/4YHXEf3fUWPxb3Vml03P105dlOdYZxWhzDleG9QDOqQ+XhMQGaJeM6aIaPKlLk43juL96z+RVggQnxN0cKk1JSvTaam6DsvQcB+IeN36/PL13oTlWTHIHchPmu2+XUS8rQ1sbiypLh7qVAk+y76+seupnWG8ImhiWyoG3SPSKeL9bzzUmdm9+OcQ2xR+xGfA3p1EbfIDmIjfqi8ml5qJpWYhmwQ9vpI7STuoU2sV2ocrqTffrgO0i40d1WEV9RnYlSbwJgohS94VHRrxfEuL2ZdZy2LfYIeCil2Jm/GwKa1yu/wAyuxCYNLYRS1PhNq2oKiMQez2bMBxLJ1qBR5pS7Z1TUg1UJo8j1FBa4FNKVpirFzYixM8jt7GQnbYkkdVDbRDt/4j62KUf3zEr9tYQJtTVm1fmDOyKpKBUsyEar26C6bWzFtknRnOXsLpjAls+rKjiPlFo2gTYXt4kmF7YC5U6bPdJBqj01pp56UpjkijoYZEXWVuzxo50mOxrtPF3s9N6CzogG5IkouUZpMHOvw1quKb8BLYKk6mx5k9JHmk13exQkaj/WRNtYmDDWaGHwm1SwUtlLgGbCZsUFXIQPEFFKJNROTBobDAQKIjVNSkbqi7yJKpMMiGqWtCAjd0YAPCwAP7nbc+0HIw0sbV7f3pySDc9qBdt0qLZqFMhIYcxZu1YRYuDxJoJ4LxtPPpDTBVFigvQuylJTc9gTLMaeJG+ss69y1pFBqUJwFd9XE51CPYp9deDr0OW6cfvOE1am2HYvKdEkbBUl4WqJUgFZCPt8jMW66YYm/gFhM2M6Z94W+68+D58rF0hZ/+itZ3/49A9YSaorjkDwRizhx0pgshL8Ow0LBOdQ0vYtmIH4vIP8klBWSgRSFMy6niVl2rCthY/GWgoRcRDHLxvXOx1XN66u6K/PL2N8Rk+2f1zjtUcel/HvZN5Bmv3ZuEn+7K3fibDoIMkwubV1lZOYmTCtQ27yMad+X96P7Yy8Xa+2x4YtL+/vIRFpbFXzhycB+cNhIH98EpA/Hgby7ElAnh0G8qcnAfnTYSCtw3kCmM7RGaAKJoUUWiSCOT8W/YSpDEm6A36AWESi9aoHm7SDavmdj4DpXyd6LYi8C4KYzYb4DYfQk2jcblXChRWRnPLgA/OCm33owGD0bo9XymxcUfii5gETHuMWFhvDx/GQ9g+cHVHytY/mLSXWcxFuqSfM6rYfuXng1OTRhiNE+JGmbS52FOPwasFePvqvdgfho6FTrLro5bgKcTXQE1AGnRvanM940EGzaSP5aKM5zBhMcCoBenrJIoXkJxT8hYNVwwwJ05njdAafucl0eh3j14sPDvOvUPJbLlZdhfv3F++rL1JONSWM/rX78NYa3ufzD//88sV826ffNqTp+PbHs88fPG2LHgpin/pkDCBZo4SzKXABZWH23b6jQKPSJhH3bYmdlK8/f722lB2ll6dnPVcaH8/OP19A6yeNkm4hxYJhPrWpMt6TvAi2r2y/Ts6bjTHLUmF6AhOdFCCVfm5TzgsBUpQaQQvIhNInMKFJXoQrEgAff+6R2c+dP2yJ5GeYXF19fN4nlp+/XF3C1s8ovyOMppsywylsR7BdpF71QH+154fnzR8aC2CbsAlj610yW3sEZy/ObMzdu1kpVUanTgU/PXtx1omlJcZXMDEB/vdXn64ve4X5qiXMV0cI8+r6apvUdoFvWwg2B2lmZN3xoEjHaBH8aEH+dPrKLjAFutz0GcUUnUo3jDICsutNkczePFENWohbcx6XlFOVdZjaftDu6zPz03G8wbUrQZZMd3uEaJgmTxk1lKweyWlgxcSS5Aa53i+9g4uNnYs9wiNJHeNOybwsVhll2J4HKIuYAxF22cOhrcdfNiMv4S6U5vxekGbjunpDqqqgb83zwQKNbze8TU0qkdlwm4Rjk9bYX5O2tXwkyapb4J4Y0fUe+OuJYLB45ISRfYxl551Ju/ehrxBgWDo2pHWv6CLBzsxJn2gixAN+8A6lHRMsOf3T3kQpmiIQN5MScx8R3reBAMbuYdVu0LxTC19H+Ku21LicrVuyyn37XosYzvc/DHwgxlsND85+EYn1cFiOhNu2DPOBznBtPg0SdZ5mbcc9E8Kr6+FAPxDbzM50nl9oPRAkWFt4HFG45fdMGocNj3/Q7U6QN3aG2jF2Cn0OZN+s5x50zddOT4aPWd2D9peuUcrNdFXCidj+rqlViNCA0ZnC3Har1eb+EMYWa5fdfAP75ZixKl+hsvxYljdMweT88uv3b/7l6k5RTUiV4Xt6HltXNZbZFUpsPJO6fxK//rto2/hHPM4jOb+vziVblrr/plUTSWAsH4ZJddwDRAz9mMuDfTnsADAMfVvGmdrL/6l3bNOqlhZ9hdf9N8DG05YRn5vXqDpbznzteZKTe/vv563eGz9YrjM7a7zszCDqPvCKkEk3XsCkijy4iLc2vf03R0nAZOTEt+E4pA+e3QwqjM2JH1lh/l3PHrGe/betY2+KwVanfT82pnBSFiedP6qLnu0fLQm1f7ZUaVGcWECpWPGTiELKRknHUfF3ZW5THZLamUqTIBgxiqUvcLnjFdWOv8erHWW1/sva11qYi7WHFhG70Z1u2IEwfaGq8fyHWDxLwrqLmEfh+Z0wdjgeZ8zGQfTOGcqCSJKj9tf1Fhe8Qb1C5PDCptp/uHfNXv3xH/4fRlB/nL4covR+FBNvPfUtF+Csp3WaMecicMcLw3Yrdah5jAwi5QA7Kr+3dtkG2KH3AwPcPgMHAew8CANDDB2KqjgXPBUuztg6G+6t/hMC0W4EIlzJIUx2upSuO59gMKlEKZMRhhlImgb+LNmGuMb7UPQS0+HuKNdPBLIM7GP32f8FAAD//xOaAGg=" } diff --git a/metricbeat/module/haproxy/info/_meta/fields.yml b/metricbeat/module/haproxy/info/_meta/fields.yml index ba7ce5a88f1..fb4474db898 100644 --- a/metricbeat/module/haproxy/info/_meta/fields.yml +++ b/metricbeat/module/haproxy/info/_meta/fields.yml @@ -153,14 +153,17 @@ - name: in type: long description: > + Incoming compressed data in bits per second. - name: out type: long description: > + Outgoing compressed data in bits per second. - name: rate_limit type: long description: > + Rate limit of compressed data in bits per second. - name: connection type: group @@ -175,14 +178,17 @@ - name: value type: long description: > + Number of connections in the last second. - name: limit type: long description: > + Rate limit of connections. - name: max type: long description: > + Maximum rate of connections. - name: current type: long @@ -221,14 +227,17 @@ - name: requests.total type: long description: > + Total number of requests. - name: sockets.max type: long description: > + Maximum number of sockets. - name: requests.max type: long description: > + Maximum number of requests. - name: pipes type: group @@ -237,14 +246,17 @@ - name: used type: integer description: > + Number of used pipes during kernel-based tcp splicing. - name: free type: integer description: > + Number of free pipes. - name: max type: integer description: > + Maximum number of used pipes. - name: session type: group @@ -253,14 +265,17 @@ - name: rate.value type: integer description: > + Rate of session per seconds. - name: rate.limit type: integer description: > + Rate limit of sessions. - name: rate.max type: integer description: > + Maximum rate of sessions. - name: ssl @@ -269,15 +284,18 @@ fields: - name: rate.value type: integer - description: + description: > + Rate of SSL requests. - name: rate.limit type: integer - description: + description: > + Rate limit of SSL requests. - name: rate.max type: integer - description: + description: > + Maximum rate of SSL requests. - name: frontend type: group @@ -285,16 +303,19 @@ fields: - name: key_rate.value type: integer - description: + description: > + Key rate of SSL frontend. - name: key_rate.max type: integer - description: + description: > + Maximum key rate of SSL frontend. - name: session_reuse.pct type: scaled_float format: percent - description: + description: > + Rate of reuse of SSL frontend sessions. - name: backend type: group @@ -302,17 +323,21 @@ fields: - name: key_rate.value type: integer - description: + description: > + Key rate of SSL backend sessions. - name: key_rate.max type: integer - description: MaxConnRate + description: > + Maximum key rate of SSL backend sessions. - name: cached_lookups type: long - description: + description: > + Number of SSL cache lookups. - name: cache_misses type: long - description: + description: > + Number of SSL cache misses. - name: zlib_mem_usage @@ -323,12 +348,15 @@ - name: value type: integer description: > + Memory usage of zlib. - name: max type: integer description: > + Maximum memory usage of zlib. - name: idle.pct type: scaled_float format: percent description: > + Percentage of idle time. diff --git a/vendor/modules.txt b/vendor/modules.txt new file mode 100644 index 00000000000..7ee98f1384e --- /dev/null +++ b/vendor/modules.txt @@ -0,0 +1,1457 @@ +# 4d63.com/embedfiles v0.0.0-20190311033909-995e0740726f +4d63.com/embedfiles +# 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b +4d63.com/tz +# cloud.google.com/go v0.51.0 +cloud.google.com/go +cloud.google.com/go/compute/metadata +cloud.google.com/go/functions/metadata +cloud.google.com/go/iam +cloud.google.com/go/internal +cloud.google.com/go/internal/fields +cloud.google.com/go/internal/optional +cloud.google.com/go/internal/trace +cloud.google.com/go/internal/version +cloud.google.com/go/monitoring/apiv3 +# cloud.google.com/go/datastore v1.0.0 +cloud.google.com/go/datastore +cloud.google.com/go/datastore/internal/gaepb +# cloud.google.com/go/pubsub v1.0.1 +cloud.google.com/go/pubsub +cloud.google.com/go/pubsub/apiv1 +cloud.google.com/go/pubsub/internal/distribution +# cloud.google.com/go/storage v1.0.0 +cloud.google.com/go/storage +# code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee +code.cloudfoundry.org/go-diodes +# code.cloudfoundry.org/go-loggregator v7.4.0+incompatible +code.cloudfoundry.org/go-loggregator +code.cloudfoundry.org/go-loggregator/conversion +code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2 +# code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f +code.cloudfoundry.org/gofileutils/fileutils +# code.cloudfoundry.org/rfc5424 v0.0.0-20180905210152-236a6d29298a +code.cloudfoundry.org/rfc5424 +# github.com/Azure/azure-amqp-common-go/v3 v3.0.0 +github.com/Azure/azure-amqp-common-go/v3 +github.com/Azure/azure-amqp-common-go/v3/aad +github.com/Azure/azure-amqp-common-go/v3/auth +github.com/Azure/azure-amqp-common-go/v3/cbs +github.com/Azure/azure-amqp-common-go/v3/conn +github.com/Azure/azure-amqp-common-go/v3/internal +github.com/Azure/azure-amqp-common-go/v3/internal/tracing +github.com/Azure/azure-amqp-common-go/v3/rpc +github.com/Azure/azure-amqp-common-go/v3/sas +github.com/Azure/azure-amqp-common-go/v3/uuid +# github.com/Azure/azure-event-hubs-go/v3 v3.1.2 +github.com/Azure/azure-event-hubs-go/v3 +github.com/Azure/azure-event-hubs-go/v3/atom +github.com/Azure/azure-event-hubs-go/v3/eph +github.com/Azure/azure-event-hubs-go/v3/persist +github.com/Azure/azure-event-hubs-go/v3/storage +# github.com/Azure/azure-pipeline-go v0.2.1 +github.com/Azure/azure-pipeline-go/pipeline +# github.com/Azure/azure-sdk-for-go v37.1.0+incompatible +github.com/Azure/azure-sdk-for-go/services/eventhub/mgmt/2017-04-01/eventhub +github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights +github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources +github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage +github.com/Azure/azure-sdk-for-go/version +# github.com/Azure/azure-storage-blob-go v0.8.0 +github.com/Azure/azure-storage-blob-go/azblob +# github.com/Azure/go-amqp v0.12.6 +github.com/Azure/go-amqp +github.com/Azure/go-amqp/internal/testconn +# github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 +github.com/Azure/go-ansiterm +github.com/Azure/go-ansiterm/winterm +# github.com/Azure/go-autorest/autorest v0.9.4 +github.com/Azure/go-autorest/autorest +github.com/Azure/go-autorest/autorest/azure +# github.com/Azure/go-autorest/autorest/adal v0.8.1 +github.com/Azure/go-autorest/autorest/adal +# github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 +github.com/Azure/go-autorest/autorest/azure/auth +# github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 +github.com/Azure/go-autorest/autorest/azure/cli +# github.com/Azure/go-autorest/autorest/date v0.2.0 +github.com/Azure/go-autorest/autorest/date +# github.com/Azure/go-autorest/autorest/to v0.3.0 +github.com/Azure/go-autorest/autorest/to +# github.com/Azure/go-autorest/autorest/validation v0.2.0 +github.com/Azure/go-autorest/autorest/validation +# github.com/Azure/go-autorest/logger v0.1.0 +github.com/Azure/go-autorest/logger +# github.com/Azure/go-autorest/tracing v0.5.0 +github.com/Azure/go-autorest/tracing +# github.com/BurntSushi/toml v0.3.1 +github.com/BurntSushi/toml +# github.com/Masterminds/semver v1.4.2 +github.com/Masterminds/semver +# github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 +github.com/Microsoft/go-winio +github.com/Microsoft/go-winio/pkg/guid +# github.com/Microsoft/hcsshim v0.8.7 +github.com/Microsoft/hcsshim/osversion +# github.com/Shopify/sarama v0.0.0-00010101000000-000000000000 => github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e +github.com/Shopify/sarama +# github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 +github.com/StackExchange/wmi +# github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc +github.com/aerospike/aerospike-client-go +github.com/aerospike/aerospike-client-go/internal/lua +github.com/aerospike/aerospike-client-go/internal/lua/resources +github.com/aerospike/aerospike-client-go/logger +github.com/aerospike/aerospike-client-go/pkg/bcrypt +github.com/aerospike/aerospike-client-go/pkg/ripemd160 +github.com/aerospike/aerospike-client-go/types +github.com/aerospike/aerospike-client-go/types/atomic +github.com/aerospike/aerospike-client-go/types/particle_type +github.com/aerospike/aerospike-client-go/types/rand +github.com/aerospike/aerospike-client-go/utils/buffer +# github.com/akavel/rsrc v0.8.0 +github.com/akavel/rsrc/binutil +github.com/akavel/rsrc/coff +github.com/akavel/rsrc/ico +# github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 +github.com/andrewkroh/sys/windows/svc/eventlog +# github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5 +github.com/antlr/antlr4/runtime/Go/antlr +# github.com/armon/go-radix v1.0.0 +github.com/armon/go-radix +# github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 +github.com/armon/go-socks5 +# github.com/aws/aws-lambda-go v1.6.0 +github.com/aws/aws-lambda-go/events +github.com/aws/aws-lambda-go/lambda +github.com/aws/aws-lambda-go/lambda/messages +github.com/aws/aws-lambda-go/lambdacontext +# github.com/aws/aws-sdk-go-v2 v0.9.0 +github.com/aws/aws-sdk-go-v2/aws +github.com/aws/aws-sdk-go-v2/aws/arn +github.com/aws/aws-sdk-go-v2/aws/awserr +github.com/aws/aws-sdk-go-v2/aws/defaults +github.com/aws/aws-sdk-go-v2/aws/ec2metadata +github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds +github.com/aws/aws-sdk-go-v2/aws/endpointcreds +github.com/aws/aws-sdk-go-v2/aws/endpoints +github.com/aws/aws-sdk-go-v2/aws/external +github.com/aws/aws-sdk-go-v2/aws/signer/v4 +github.com/aws/aws-sdk-go-v2/aws/stscreds +github.com/aws/aws-sdk-go-v2/internal/awsutil +github.com/aws/aws-sdk-go-v2/internal/ini +github.com/aws/aws-sdk-go-v2/internal/sdk +github.com/aws/aws-sdk-go-v2/private/protocol +github.com/aws/aws-sdk-go-v2/private/protocol/ec2query +github.com/aws/aws-sdk-go-v2/private/protocol/json/jsonutil +github.com/aws/aws-sdk-go-v2/private/protocol/jsonrpc +github.com/aws/aws-sdk-go-v2/private/protocol/query +github.com/aws/aws-sdk-go-v2/private/protocol/query/queryutil +github.com/aws/aws-sdk-go-v2/private/protocol/rest +github.com/aws/aws-sdk-go-v2/private/protocol/restxml +github.com/aws/aws-sdk-go-v2/private/protocol/xml +github.com/aws/aws-sdk-go-v2/private/protocol/xml/xmlutil +github.com/aws/aws-sdk-go-v2/service/cloudformation +github.com/aws/aws-sdk-go-v2/service/cloudformation/cloudformationiface +github.com/aws/aws-sdk-go-v2/service/cloudwatch +github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface +github.com/aws/aws-sdk-go-v2/service/ec2 +github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface +github.com/aws/aws-sdk-go-v2/service/iam +github.com/aws/aws-sdk-go-v2/service/rds +github.com/aws/aws-sdk-go-v2/service/rds/rdsiface +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface +github.com/aws/aws-sdk-go-v2/service/s3 +github.com/aws/aws-sdk-go-v2/service/s3/s3iface +github.com/aws/aws-sdk-go-v2/service/sqs +github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface +github.com/aws/aws-sdk-go-v2/service/sts +# github.com/awslabs/goformation/v4 v4.1.0 +github.com/awslabs/goformation/v4/cloudformation +github.com/awslabs/goformation/v4/cloudformation/accessanalyzer +github.com/awslabs/goformation/v4/cloudformation/amazonmq +github.com/awslabs/goformation/v4/cloudformation/amplify +github.com/awslabs/goformation/v4/cloudformation/apigateway +github.com/awslabs/goformation/v4/cloudformation/apigatewayv2 +github.com/awslabs/goformation/v4/cloudformation/applicationautoscaling +github.com/awslabs/goformation/v4/cloudformation/appmesh +github.com/awslabs/goformation/v4/cloudformation/appstream +github.com/awslabs/goformation/v4/cloudformation/appsync +github.com/awslabs/goformation/v4/cloudformation/ask +github.com/awslabs/goformation/v4/cloudformation/athena +github.com/awslabs/goformation/v4/cloudformation/autoscaling +github.com/awslabs/goformation/v4/cloudformation/autoscalingplans +github.com/awslabs/goformation/v4/cloudformation/backup +github.com/awslabs/goformation/v4/cloudformation/batch +github.com/awslabs/goformation/v4/cloudformation/budgets +github.com/awslabs/goformation/v4/cloudformation/certificatemanager +github.com/awslabs/goformation/v4/cloudformation/cloud9 +github.com/awslabs/goformation/v4/cloudformation/cloudformation +github.com/awslabs/goformation/v4/cloudformation/cloudfront +github.com/awslabs/goformation/v4/cloudformation/cloudtrail +github.com/awslabs/goformation/v4/cloudformation/cloudwatch +github.com/awslabs/goformation/v4/cloudformation/codebuild +github.com/awslabs/goformation/v4/cloudformation/codecommit +github.com/awslabs/goformation/v4/cloudformation/codedeploy +github.com/awslabs/goformation/v4/cloudformation/codepipeline +github.com/awslabs/goformation/v4/cloudformation/codestar +github.com/awslabs/goformation/v4/cloudformation/codestarnotifications +github.com/awslabs/goformation/v4/cloudformation/cognito +github.com/awslabs/goformation/v4/cloudformation/config +github.com/awslabs/goformation/v4/cloudformation/datapipeline +github.com/awslabs/goformation/v4/cloudformation/dax +github.com/awslabs/goformation/v4/cloudformation/directoryservice +github.com/awslabs/goformation/v4/cloudformation/dlm +github.com/awslabs/goformation/v4/cloudformation/dms +github.com/awslabs/goformation/v4/cloudformation/docdb +github.com/awslabs/goformation/v4/cloudformation/dynamodb +github.com/awslabs/goformation/v4/cloudformation/ec2 +github.com/awslabs/goformation/v4/cloudformation/ecr +github.com/awslabs/goformation/v4/cloudformation/ecs +github.com/awslabs/goformation/v4/cloudformation/efs +github.com/awslabs/goformation/v4/cloudformation/eks +github.com/awslabs/goformation/v4/cloudformation/elasticache +github.com/awslabs/goformation/v4/cloudformation/elasticbeanstalk +github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancing +github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2 +github.com/awslabs/goformation/v4/cloudformation/elasticsearch +github.com/awslabs/goformation/v4/cloudformation/emr +github.com/awslabs/goformation/v4/cloudformation/events +github.com/awslabs/goformation/v4/cloudformation/eventschemas +github.com/awslabs/goformation/v4/cloudformation/fsx +github.com/awslabs/goformation/v4/cloudformation/gamelift +github.com/awslabs/goformation/v4/cloudformation/glue +github.com/awslabs/goformation/v4/cloudformation/greengrass +github.com/awslabs/goformation/v4/cloudformation/guardduty +github.com/awslabs/goformation/v4/cloudformation/iam +github.com/awslabs/goformation/v4/cloudformation/inspector +github.com/awslabs/goformation/v4/cloudformation/iot +github.com/awslabs/goformation/v4/cloudformation/iot1click +github.com/awslabs/goformation/v4/cloudformation/iotanalytics +github.com/awslabs/goformation/v4/cloudformation/iotevents +github.com/awslabs/goformation/v4/cloudformation/iotthingsgraph +github.com/awslabs/goformation/v4/cloudformation/kinesis +github.com/awslabs/goformation/v4/cloudformation/kinesisanalytics +github.com/awslabs/goformation/v4/cloudformation/kinesisanalyticsv2 +github.com/awslabs/goformation/v4/cloudformation/kinesisfirehose +github.com/awslabs/goformation/v4/cloudformation/kms +github.com/awslabs/goformation/v4/cloudformation/lakeformation +github.com/awslabs/goformation/v4/cloudformation/lambda +github.com/awslabs/goformation/v4/cloudformation/logs +github.com/awslabs/goformation/v4/cloudformation/managedblockchain +github.com/awslabs/goformation/v4/cloudformation/mediaconvert +github.com/awslabs/goformation/v4/cloudformation/medialive +github.com/awslabs/goformation/v4/cloudformation/mediastore +github.com/awslabs/goformation/v4/cloudformation/msk +github.com/awslabs/goformation/v4/cloudformation/neptune +github.com/awslabs/goformation/v4/cloudformation/opsworks +github.com/awslabs/goformation/v4/cloudformation/opsworkscm +github.com/awslabs/goformation/v4/cloudformation/pinpoint +github.com/awslabs/goformation/v4/cloudformation/pinpointemail +github.com/awslabs/goformation/v4/cloudformation/policies +github.com/awslabs/goformation/v4/cloudformation/qldb +github.com/awslabs/goformation/v4/cloudformation/ram +github.com/awslabs/goformation/v4/cloudformation/rds +github.com/awslabs/goformation/v4/cloudformation/redshift +github.com/awslabs/goformation/v4/cloudformation/robomaker +github.com/awslabs/goformation/v4/cloudformation/route53 +github.com/awslabs/goformation/v4/cloudformation/route53resolver +github.com/awslabs/goformation/v4/cloudformation/s3 +github.com/awslabs/goformation/v4/cloudformation/sagemaker +github.com/awslabs/goformation/v4/cloudformation/sdb +github.com/awslabs/goformation/v4/cloudformation/secretsmanager +github.com/awslabs/goformation/v4/cloudformation/securityhub +github.com/awslabs/goformation/v4/cloudformation/serverless +github.com/awslabs/goformation/v4/cloudformation/servicecatalog +github.com/awslabs/goformation/v4/cloudformation/servicediscovery +github.com/awslabs/goformation/v4/cloudformation/ses +github.com/awslabs/goformation/v4/cloudformation/sns +github.com/awslabs/goformation/v4/cloudformation/sqs +github.com/awslabs/goformation/v4/cloudformation/ssm +github.com/awslabs/goformation/v4/cloudformation/stepfunctions +github.com/awslabs/goformation/v4/cloudformation/tags +github.com/awslabs/goformation/v4/cloudformation/transfer +github.com/awslabs/goformation/v4/cloudformation/utils +github.com/awslabs/goformation/v4/cloudformation/waf +github.com/awslabs/goformation/v4/cloudformation/wafregional +github.com/awslabs/goformation/v4/cloudformation/wafv2 +github.com/awslabs/goformation/v4/cloudformation/workspaces +github.com/awslabs/goformation/v4/intrinsics +# github.com/beorn7/perks v1.0.1 +github.com/beorn7/perks/quantile +# github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 +github.com/blakesmith/ar +# github.com/bradleyfalzon/ghinstallation v1.1.0 +github.com/bradleyfalzon/ghinstallation +# github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible +github.com/bsm/sarama-cluster +# github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e +github.com/cavaliercoder/go-rpm +github.com/cavaliercoder/go-rpm/version +# github.com/cespare/xxhash/v2 v2.1.1 +github.com/cespare/xxhash/v2 +# github.com/cloudfoundry-community/go-cfclient v0.0.0-20190808214049-35bcce23fc5f +github.com/cloudfoundry-community/go-cfclient +# github.com/cloudfoundry/noaa v2.1.0+incompatible +github.com/cloudfoundry/noaa +github.com/cloudfoundry/noaa/consumer +github.com/cloudfoundry/noaa/consumer/internal +github.com/cloudfoundry/noaa/errors +# github.com/cloudfoundry/sonde-go v0.0.0-20171206171820-b33733203bb4 +github.com/cloudfoundry/sonde-go/events +# github.com/containerd/containerd v1.3.3 +github.com/containerd/containerd/errdefs +# github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 +github.com/containerd/continuity/fs +github.com/containerd/continuity/pathdriver +github.com/containerd/continuity/syscallx +github.com/containerd/continuity/sysx +# github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c +github.com/containerd/fifo +# github.com/coreos/bbolt v1.3.1-coreos.6.0.20180318001526-af9db2027c98 +github.com/coreos/bbolt +# github.com/coreos/go-systemd/v22 v22.0.0 +github.com/coreos/go-systemd/v22/activation +github.com/coreos/go-systemd/v22/dbus +github.com/coreos/go-systemd/v22/internal/dlopen +github.com/coreos/go-systemd/v22/sdjournal +# github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea +github.com/coreos/pkg/dlopen +# github.com/davecgh/go-spew v1.1.1 +github.com/davecgh/go-spew/spew +# github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 +github.com/davecgh/go-xdr/xdr2 +# github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e +github.com/denisenkom/go-mssqldb +github.com/denisenkom/go-mssqldb/internal/cp +github.com/denisenkom/go-mssqldb/internal/decimal +github.com/denisenkom/go-mssqldb/internal/querytext +# github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 +github.com/devigned/tab +# github.com/dgrijalva/jwt-go v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible +github.com/dgrijalva/jwt-go +# github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1 +github.com/digitalocean/go-libvirt +github.com/digitalocean/go-libvirt/internal/constants +github.com/digitalocean/go-libvirt/libvirttest +# github.com/dimchansky/utfbom v1.1.0 +github.com/dimchansky/utfbom +# github.com/dlclark/regexp2 v1.1.7-0.20171009020623-7632a260cbaf +github.com/dlclark/regexp2 +github.com/dlclark/regexp2/syntax +# github.com/docker/distribution v2.7.1+incompatible +github.com/docker/distribution/digestset +github.com/docker/distribution/reference +github.com/docker/distribution/registry/api/errcode +# github.com/docker/docker v1.4.2-0.20170802015333-8af4db6f002a => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 +github.com/docker/docker/api +github.com/docker/docker/api/types +github.com/docker/docker/api/types/backend +github.com/docker/docker/api/types/blkiodev +github.com/docker/docker/api/types/container +github.com/docker/docker/api/types/events +github.com/docker/docker/api/types/filters +github.com/docker/docker/api/types/image +github.com/docker/docker/api/types/mount +github.com/docker/docker/api/types/network +github.com/docker/docker/api/types/plugins/logdriver +github.com/docker/docker/api/types/registry +github.com/docker/docker/api/types/strslice +github.com/docker/docker/api/types/swarm +github.com/docker/docker/api/types/swarm/runtime +github.com/docker/docker/api/types/time +github.com/docker/docker/api/types/versions +github.com/docker/docker/api/types/volume +github.com/docker/docker/client +github.com/docker/docker/daemon/logger +github.com/docker/docker/errdefs +github.com/docker/docker/pkg/archive +github.com/docker/docker/pkg/fileutils +github.com/docker/docker/pkg/idtools +github.com/docker/docker/pkg/ioutils +github.com/docker/docker/pkg/jsonmessage +github.com/docker/docker/pkg/longpath +github.com/docker/docker/pkg/mount +github.com/docker/docker/pkg/plugingetter +github.com/docker/docker/pkg/plugins +github.com/docker/docker/pkg/plugins/transport +github.com/docker/docker/pkg/pools +github.com/docker/docker/pkg/progress +github.com/docker/docker/pkg/streamformatter +github.com/docker/docker/pkg/stringid +github.com/docker/docker/pkg/system +github.com/docker/docker/pkg/term +github.com/docker/docker/pkg/term/windows +# github.com/docker/go-connections v0.4.0 +github.com/docker/go-connections/nat +github.com/docker/go-connections/sockets +github.com/docker/go-connections/tlsconfig +# github.com/docker/go-metrics v0.0.1 +github.com/docker/go-metrics +# github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8 => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f +github.com/docker/go-plugins-helpers/sdk +# github.com/docker/go-units v0.4.0 +github.com/docker/go-units +# github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 +github.com/docker/spdystream +github.com/docker/spdystream/spdy +# github.com/dop251/goja v0.0.0-00010101000000-000000000000 => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 +github.com/dop251/goja +github.com/dop251/goja/ast +github.com/dop251/goja/file +github.com/dop251/goja/parser +github.com/dop251/goja/token +# github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 +github.com/dop251/goja_nodejs/require +github.com/dop251/goja_nodejs/util +# github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 +github.com/dustin/go-humanize +# github.com/eapache/go-resiliency v1.2.0 +github.com/eapache/go-resiliency/breaker +# github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 +github.com/eapache/go-xerial-snappy +# github.com/eapache/queue v1.1.0 +github.com/eapache/queue +# github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 +github.com/eclipse/paho.mqtt.golang +github.com/eclipse/paho.mqtt.golang/packets +# github.com/elastic/ecs v1.5.0 +github.com/elastic/ecs/code/go/ecs +# github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07 +github.com/elastic/elastic-agent-client/v7/pkg/client +github.com/elastic/elastic-agent-client/v7/pkg/proto +github.com/elastic/elastic-agent-client/v7/pkg/utils +# github.com/elastic/go-concert v0.0.2 +github.com/elastic/go-concert +github.com/elastic/go-concert/atomic +github.com/elastic/go-concert/chorus +github.com/elastic/go-concert/unison +# github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 +github.com/elastic/go-libaudit/v2 +github.com/elastic/go-libaudit/v2/aucoalesce +github.com/elastic/go-libaudit/v2/auparse +github.com/elastic/go-libaudit/v2/rule +github.com/elastic/go-libaudit/v2/rule/flags +github.com/elastic/go-libaudit/v2/sys +# github.com/elastic/go-licenser v0.2.1 +github.com/elastic/go-licenser +github.com/elastic/go-licenser/licensing +# github.com/elastic/go-lookslike v0.3.0 +github.com/elastic/go-lookslike +github.com/elastic/go-lookslike/internal/llreflect +github.com/elastic/go-lookslike/isdef +github.com/elastic/go-lookslike/llpath +github.com/elastic/go-lookslike/llresult +github.com/elastic/go-lookslike/testslike +github.com/elastic/go-lookslike/validator +# github.com/elastic/go-lumber v0.1.0 +github.com/elastic/go-lumber/client/v2 +github.com/elastic/go-lumber/lj +github.com/elastic/go-lumber/log +github.com/elastic/go-lumber/protocol/v2 +github.com/elastic/go-lumber/server/internal +github.com/elastic/go-lumber/server/v2 +# github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 +github.com/elastic/go-perf +# github.com/elastic/go-seccomp-bpf v1.1.0 +github.com/elastic/go-seccomp-bpf +github.com/elastic/go-seccomp-bpf/arch +# github.com/elastic/go-structform v0.0.7 +github.com/elastic/go-structform +github.com/elastic/go-structform/cborl +github.com/elastic/go-structform/gotype +github.com/elastic/go-structform/internal/unsafe +github.com/elastic/go-structform/json +github.com/elastic/go-structform/ubjson +github.com/elastic/go-structform/visitors +# github.com/elastic/go-sysinfo v1.3.0 +github.com/elastic/go-sysinfo +github.com/elastic/go-sysinfo/internal/registry +github.com/elastic/go-sysinfo/providers/darwin +github.com/elastic/go-sysinfo/providers/linux +github.com/elastic/go-sysinfo/providers/shared +github.com/elastic/go-sysinfo/providers/windows +github.com/elastic/go-sysinfo/types +# github.com/elastic/go-txfile v0.0.7 +github.com/elastic/go-txfile +github.com/elastic/go-txfile/dev-tools/lib/mage/xbuild +github.com/elastic/go-txfile/internal/cleanup +github.com/elastic/go-txfile/internal/invariant +github.com/elastic/go-txfile/internal/iter +github.com/elastic/go-txfile/internal/strbld +github.com/elastic/go-txfile/internal/tracelog +github.com/elastic/go-txfile/internal/vfs +github.com/elastic/go-txfile/internal/vfs/osfs +github.com/elastic/go-txfile/internal/vfs/osfs/osfstest +github.com/elastic/go-txfile/pq +github.com/elastic/go-txfile/txerr +github.com/elastic/go-txfile/txfiletest +# github.com/elastic/go-ucfg v0.8.3 +github.com/elastic/go-ucfg +github.com/elastic/go-ucfg/cfgutil +github.com/elastic/go-ucfg/flag +github.com/elastic/go-ucfg/json +github.com/elastic/go-ucfg/parse +github.com/elastic/go-ucfg/yaml +# github.com/elastic/go-windows v1.0.1 +github.com/elastic/go-windows +# github.com/elastic/gosigar v0.10.5 +github.com/elastic/gosigar +github.com/elastic/gosigar/cgroup +github.com/elastic/gosigar/sys +github.com/elastic/gosigar/sys/linux +github.com/elastic/gosigar/sys/windows +# github.com/evanphx/json-patch v4.2.0+incompatible +github.com/evanphx/json-patch +# github.com/fatih/color v1.5.0 +github.com/fatih/color +# github.com/fsnotify/fsevents v0.0.0-00010101000000-000000000000 => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 +github.com/fsnotify/fsevents +# github.com/fsnotify/fsnotify v1.4.7 => github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d +github.com/fsnotify/fsnotify +# github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24 +github.com/garyburd/redigo/internal +github.com/garyburd/redigo/redis +# github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 +github.com/go-ole/go-ole +github.com/go-ole/go-ole/oleutil +# github.com/go-sourcemap/sourcemap v2.1.2+incompatible +github.com/go-sourcemap/sourcemap +github.com/go-sourcemap/sourcemap/internal/base64vlq +# github.com/go-sql-driver/mysql v1.4.1 +github.com/go-sql-driver/mysql +# github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be +github.com/gocarina/gocsv +# github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e +github.com/godbus/dbus +# github.com/godbus/dbus/v5 v5.0.3 +github.com/godbus/dbus/v5 +# github.com/godror/godror v0.10.4 +github.com/godror/godror +# github.com/gofrs/flock v0.7.2-0.20190320160742-5135e617513b +github.com/gofrs/flock +# github.com/gofrs/uuid v3.3.0+incompatible +github.com/gofrs/uuid +# github.com/gogo/protobuf v1.3.1 +github.com/gogo/protobuf/gogoproto +github.com/gogo/protobuf/proto +github.com/gogo/protobuf/protoc-gen-gogo/descriptor +github.com/gogo/protobuf/sortkeys +github.com/gogo/protobuf/types +# github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe +github.com/golang-sql/civil +# github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 +github.com/golang/groupcache/lru +# github.com/golang/protobuf v1.4.2 +github.com/golang/protobuf/descriptor +github.com/golang/protobuf/internal/gengogrpc +github.com/golang/protobuf/jsonpb +github.com/golang/protobuf/proto +github.com/golang/protobuf/protoc-gen-go +github.com/golang/protobuf/protoc-gen-go/descriptor +github.com/golang/protobuf/ptypes +github.com/golang/protobuf/ptypes/any +github.com/golang/protobuf/ptypes/duration +github.com/golang/protobuf/ptypes/empty +github.com/golang/protobuf/ptypes/struct +github.com/golang/protobuf/ptypes/timestamp +github.com/golang/protobuf/ptypes/wrappers +# github.com/golang/snappy v0.0.1 +github.com/golang/snappy +# github.com/google/flatbuffers v1.7.2-0.20170925184458-7a6b2bf521e9 +github.com/google/flatbuffers/go +# github.com/google/go-cmp v0.4.0 +github.com/google/go-cmp/cmp +github.com/google/go-cmp/cmp/internal/diff +github.com/google/go-cmp/cmp/internal/flags +github.com/google/go-cmp/cmp/internal/function +github.com/google/go-cmp/cmp/internal/value +# github.com/google/go-github/v28 v28.1.1 +github.com/google/go-github/v28/github +# github.com/google/go-github/v29 v29.0.2 +github.com/google/go-github/v29/github +# github.com/google/go-querystring v1.0.0 +github.com/google/go-querystring/query +# github.com/google/gofuzz v1.1.0 +github.com/google/gofuzz +# github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 +github.com/google/gopacket +github.com/google/gopacket/afpacket +github.com/google/gopacket/layers +# github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f +github.com/google/uuid +# github.com/googleapis/gax-go/v2 v2.0.5 +github.com/googleapis/gax-go/v2 +# github.com/googleapis/gnostic v0.3.1-0.20190624222214-25d8b0b66985 +github.com/googleapis/gnostic/OpenAPIv2 +github.com/googleapis/gnostic/compiler +github.com/googleapis/gnostic/extensions +# github.com/gorhill/cronexpr v0.0.0-20161205141322-d520615e531a +github.com/gorhill/cronexpr +# github.com/gorilla/websocket v1.4.1 +github.com/gorilla/websocket +# github.com/grpc-ecosystem/grpc-gateway v1.13.0 +github.com/grpc-ecosystem/grpc-gateway/internal +github.com/grpc-ecosystem/grpc-gateway/runtime +github.com/grpc-ecosystem/grpc-gateway/utilities +# github.com/h2non/filetype v1.0.12 +github.com/h2non/filetype +github.com/h2non/filetype/matchers +github.com/h2non/filetype/matchers/isobmff +github.com/h2non/filetype/types +# github.com/hashicorp/errwrap v1.0.0 +github.com/hashicorp/errwrap +# github.com/hashicorp/go-cleanhttp v0.5.1 +github.com/hashicorp/go-cleanhttp +# github.com/hashicorp/go-multierror v1.1.0 +github.com/hashicorp/go-multierror +# github.com/hashicorp/go-retryablehttp v0.6.6 +github.com/hashicorp/go-retryablehttp +# github.com/hashicorp/go-uuid v1.0.2 +github.com/hashicorp/go-uuid +# github.com/hashicorp/go-version v1.0.0 +github.com/hashicorp/go-version +# github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d +github.com/hashicorp/golang-lru +github.com/hashicorp/golang-lru/simplelru +# github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01 +github.com/haya14busa/go-actions-toolkit/core +# github.com/imdario/mergo v0.3.6 +github.com/imdario/mergo +# github.com/inconshreveable/mousetrap v1.0.0 +github.com/inconshreveable/mousetrap +# github.com/insomniacslk/dhcp v0.0.0-20180716145214-633285ba52b2 => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 +github.com/insomniacslk/dhcp/dhcpv4 +github.com/insomniacslk/dhcp/iana +# github.com/jcmturner/gofork v1.0.0 +github.com/jcmturner/gofork/encoding/asn1 +github.com/jcmturner/gofork/x/crypto/pbkdf2 +# github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af +github.com/jmespath/go-jmespath +# github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 +github.com/jmoiron/sqlx +github.com/jmoiron/sqlx/reflectx +# github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 +github.com/joeshaw/multierror +# github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd +github.com/josephspurrier/goversioninfo +# github.com/jpillora/backoff v1.0.0 +github.com/jpillora/backoff +# github.com/json-iterator/go v1.1.8 +github.com/json-iterator/go +# github.com/jstemmer/go-junit-report v0.9.1 +github.com/jstemmer/go-junit-report +github.com/jstemmer/go-junit-report/formatter +github.com/jstemmer/go-junit-report/parser +# github.com/klauspost/compress v1.9.8 +github.com/klauspost/compress/flate +github.com/klauspost/compress/fse +github.com/klauspost/compress/huff0 +github.com/klauspost/compress/snappy +github.com/klauspost/compress/zlib +github.com/klauspost/compress/zstd +github.com/klauspost/compress/zstd/internal/xxhash +# github.com/konsorten/go-windows-terminal-sequences v1.0.2 +github.com/konsorten/go-windows-terminal-sequences +# github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01 +github.com/lib/pq +github.com/lib/pq/oid +github.com/lib/pq/scram +# github.com/magefile/mage v1.9.0 +github.com/magefile/mage +github.com/magefile/mage/internal +github.com/magefile/mage/mage +github.com/magefile/mage/mg +github.com/magefile/mage/parse +github.com/magefile/mage/sh +github.com/magefile/mage/target +# github.com/mailru/easyjson v0.7.1 +github.com/mailru/easyjson +github.com/mailru/easyjson/buffer +github.com/mailru/easyjson/jlexer +github.com/mailru/easyjson/jwriter +# github.com/mattn/go-colorable v0.0.8 +github.com/mattn/go-colorable +# github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe +github.com/mattn/go-ieproxy +# github.com/mattn/go-isatty v0.0.2 +github.com/mattn/go-isatty +# github.com/mattn/go-shellwords v1.0.7 +github.com/mattn/go-shellwords +# github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 +github.com/matttproud/golang_protobuf_extensions/pbutil +# github.com/miekg/dns v1.1.15 +github.com/miekg/dns +# github.com/mitchellh/go-homedir v1.1.0 +github.com/mitchellh/go-homedir +# github.com/mitchellh/gox v1.0.1 +github.com/mitchellh/gox +# github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 +github.com/mitchellh/hashstructure +# github.com/mitchellh/iochan v1.0.0 +github.com/mitchellh/iochan +# github.com/mitchellh/mapstructure v1.1.2 +github.com/mitchellh/mapstructure +# github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd +github.com/modern-go/concurrent +# github.com/modern-go/reflect2 v1.0.1 +github.com/modern-go/reflect2 +# github.com/morikuni/aec v1.0.0 +github.com/morikuni/aec +# github.com/oklog/ulid v1.3.1 +github.com/oklog/ulid +# github.com/opencontainers/go-digest v1.0.0-rc1.0.20190228220655-ac19fd6e7483 +github.com/opencontainers/go-digest +# github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 +github.com/opencontainers/image-spec/specs-go +github.com/opencontainers/image-spec/specs-go/v1 +# github.com/opencontainers/runc v1.0.0-rc9 +github.com/opencontainers/runc/libcontainer/system +github.com/opencontainers/runc/libcontainer/user +# github.com/pierrec/lz4 v2.4.1+incompatible +github.com/pierrec/lz4 +github.com/pierrec/lz4/internal/xxh32 +# github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338 +github.com/pierrre/gotestcover +# github.com/pkg/errors v0.9.1 +github.com/pkg/errors +# github.com/pmezard/go-difflib v1.0.0 +github.com/pmezard/go-difflib/difflib +# github.com/prometheus/client_golang v1.1.1-0.20190913103102-20428fa0bffc +github.com/prometheus/client_golang/prometheus +github.com/prometheus/client_golang/prometheus/internal +github.com/prometheus/client_golang/prometheus/promhttp +# github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 +github.com/prometheus/client_model/go +# github.com/prometheus/common v0.7.0 +github.com/prometheus/common/expfmt +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg +github.com/prometheus/common/model +# github.com/prometheus/procfs v0.0.11 +github.com/prometheus/procfs +github.com/prometheus/procfs/internal/fs +github.com/prometheus/procfs/internal/util +# github.com/prometheus/prometheus v2.5.0+incompatible +github.com/prometheus/prometheus/prompb +# github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 +github.com/rcrowley/go-metrics +# github.com/reviewdog/errorformat v0.0.0-20200109134752-8983be9bc7dd +github.com/reviewdog/errorformat +github.com/reviewdog/errorformat/fmts +# github.com/reviewdog/reviewdog v0.9.17 +github.com/reviewdog/reviewdog +github.com/reviewdog/reviewdog/cienv +github.com/reviewdog/reviewdog/cmd/reviewdog +github.com/reviewdog/reviewdog/commands +github.com/reviewdog/reviewdog/diff +github.com/reviewdog/reviewdog/doghouse +github.com/reviewdog/reviewdog/doghouse/client +github.com/reviewdog/reviewdog/doghouse/server +github.com/reviewdog/reviewdog/doghouse/server/storage +github.com/reviewdog/reviewdog/project +github.com/reviewdog/reviewdog/service/github +github.com/reviewdog/reviewdog/service/github/githubutils +github.com/reviewdog/reviewdog/service/gitlab +github.com/reviewdog/reviewdog/service/serviceutil +# github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e +github.com/samuel/go-parser/parser +# github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 +github.com/samuel/go-thrift/parser +# github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b +github.com/sanathkr/go-yaml +# github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 +github.com/sanathkr/yaml +# github.com/santhosh-tekuri/jsonschema v1.2.4 +github.com/santhosh-tekuri/jsonschema +github.com/santhosh-tekuri/jsonschema/decoders +github.com/santhosh-tekuri/jsonschema/formats +github.com/santhosh-tekuri/jsonschema/loader +github.com/santhosh-tekuri/jsonschema/mediatypes +# github.com/shirou/gopsutil v2.19.11+incompatible +github.com/shirou/gopsutil/disk +github.com/shirou/gopsutil/internal/common +github.com/shirou/gopsutil/net +# github.com/sirupsen/logrus v1.4.2 +github.com/sirupsen/logrus +# github.com/spf13/cobra v0.0.3 +github.com/spf13/cobra +# github.com/spf13/pflag v1.0.5 +github.com/spf13/pflag +# github.com/stretchr/objx v0.2.0 +github.com/stretchr/objx +# github.com/stretchr/testify v1.5.1 +github.com/stretchr/testify/assert +github.com/stretchr/testify/mock +github.com/stretchr/testify/require +github.com/stretchr/testify/suite +# github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b +github.com/tsg/go-daemon +# github.com/tsg/gopacket v0.0.0-20190320122513-dd3d0e41124a +github.com/tsg/gopacket +github.com/tsg/gopacket/afpacket +github.com/tsg/gopacket/layers +github.com/tsg/gopacket/pcap +# github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 +github.com/urso/diag +github.com/urso/diag/ctxfmt +# github.com/urso/go-bin v0.0.0-20180220135811-781c575c9f0e +github.com/urso/go-bin +# github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a +github.com/urso/magetools/clitool +github.com/urso/magetools/ctrl +github.com/urso/magetools/fs +github.com/urso/magetools/gotool +github.com/urso/magetools/mgenv +# github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec +github.com/urso/sderr +# github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 +github.com/vmware/govmomi +github.com/vmware/govmomi/find +github.com/vmware/govmomi/list +github.com/vmware/govmomi/nfc +github.com/vmware/govmomi/object +github.com/vmware/govmomi/property +github.com/vmware/govmomi/session +github.com/vmware/govmomi/simulator +github.com/vmware/govmomi/simulator/esx +github.com/vmware/govmomi/simulator/vpx +github.com/vmware/govmomi/task +github.com/vmware/govmomi/view +github.com/vmware/govmomi/vim25 +github.com/vmware/govmomi/vim25/debug +github.com/vmware/govmomi/vim25/methods +github.com/vmware/govmomi/vim25/mo +github.com/vmware/govmomi/vim25/progress +github.com/vmware/govmomi/vim25/soap +github.com/vmware/govmomi/vim25/types +github.com/vmware/govmomi/vim25/xml +# github.com/xanzy/go-gitlab v0.22.3 +github.com/xanzy/go-gitlab +# github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c +github.com/xdg/scram +# github.com/xdg/stringprep v1.0.0 +github.com/xdg/stringprep +# github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 +github.com/yuin/gopher-lua +github.com/yuin/gopher-lua/ast +github.com/yuin/gopher-lua/parse +github.com/yuin/gopher-lua/pm +# go.elastic.co/apm v1.7.2 +go.elastic.co/apm +go.elastic.co/apm/apmconfig +go.elastic.co/apm/apmtest +go.elastic.co/apm/internal/apmcontext +go.elastic.co/apm/internal/apmhostutil +go.elastic.co/apm/internal/apmhttputil +go.elastic.co/apm/internal/apmlog +go.elastic.co/apm/internal/apmschema +go.elastic.co/apm/internal/apmstrings +go.elastic.co/apm/internal/apmversion +go.elastic.co/apm/internal/configutil +go.elastic.co/apm/internal/iochan +go.elastic.co/apm/internal/pkgerrorsutil +go.elastic.co/apm/internal/ringbuffer +go.elastic.co/apm/internal/wildcard +go.elastic.co/apm/model +go.elastic.co/apm/stacktrace +go.elastic.co/apm/transport +go.elastic.co/apm/transport/transporttest +# go.elastic.co/apm/module/apmelasticsearch v1.7.2 +go.elastic.co/apm/module/apmelasticsearch +# go.elastic.co/apm/module/apmhttp v1.7.2 +go.elastic.co/apm/module/apmhttp +# go.elastic.co/ecszap v0.2.0 +go.elastic.co/ecszap +go.elastic.co/ecszap/internal +# go.elastic.co/fastjson v1.0.0 +go.elastic.co/fastjson +# go.opencensus.io v0.22.2 +go.opencensus.io +go.opencensus.io/internal +go.opencensus.io/internal/tagencoding +go.opencensus.io/metric/metricdata +go.opencensus.io/metric/metricproducer +go.opencensus.io/plugin/ocgrpc +go.opencensus.io/plugin/ochttp +go.opencensus.io/plugin/ochttp/propagation/b3 +go.opencensus.io/resource +go.opencensus.io/stats +go.opencensus.io/stats/internal +go.opencensus.io/stats/view +go.opencensus.io/tag +go.opencensus.io/trace +go.opencensus.io/trace/internal +go.opencensus.io/trace/propagation +go.opencensus.io/trace/tracestate +# go.uber.org/atomic v1.5.0 +go.uber.org/atomic +# go.uber.org/multierr v1.3.0 +go.uber.org/multierr +# go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee +go.uber.org/tools/update-license +# go.uber.org/zap v1.14.0 +go.uber.org/zap +go.uber.org/zap/buffer +go.uber.org/zap/internal/bufferpool +go.uber.org/zap/internal/color +go.uber.org/zap/internal/exit +go.uber.org/zap/zapcore +go.uber.org/zap/zaptest/observer +# golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 +golang.org/x/crypto/blake2b +golang.org/x/crypto/blowfish +golang.org/x/crypto/cast5 +golang.org/x/crypto/chacha20 +golang.org/x/crypto/curve25519 +golang.org/x/crypto/ed25519 +golang.org/x/crypto/ed25519/internal/edwards25519 +golang.org/x/crypto/internal/subtle +golang.org/x/crypto/md4 +golang.org/x/crypto/openpgp +golang.org/x/crypto/openpgp/armor +golang.org/x/crypto/openpgp/elgamal +golang.org/x/crypto/openpgp/errors +golang.org/x/crypto/openpgp/packet +golang.org/x/crypto/openpgp/s2k +golang.org/x/crypto/pbkdf2 +golang.org/x/crypto/pkcs12 +golang.org/x/crypto/pkcs12/internal/rc2 +golang.org/x/crypto/poly1305 +golang.org/x/crypto/sha3 +golang.org/x/crypto/ssh +golang.org/x/crypto/ssh/internal/bcrypt_pbkdf +golang.org/x/crypto/ssh/terminal +# golang.org/x/exp v0.0.0-20191227195350-da58074b4299 +golang.org/x/exp/apidiff +golang.org/x/exp/cmd/apidiff +# golang.org/x/lint v0.0.0-20200130185559-910be7a94367 +golang.org/x/lint +golang.org/x/lint/golint +# golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee +golang.org/x/mod/module +golang.org/x/mod/semver +# golang.org/x/net v0.0.0-20200202094626-16171245cfb2 +golang.org/x/net/bpf +golang.org/x/net/context +golang.org/x/net/context/ctxhttp +golang.org/x/net/http/httpguts +golang.org/x/net/http/httpproxy +golang.org/x/net/http2 +golang.org/x/net/http2/hpack +golang.org/x/net/icmp +golang.org/x/net/idna +golang.org/x/net/internal/iana +golang.org/x/net/internal/socket +golang.org/x/net/internal/socks +golang.org/x/net/internal/timeseries +golang.org/x/net/ipv4 +golang.org/x/net/ipv6 +golang.org/x/net/netutil +golang.org/x/net/proxy +golang.org/x/net/publicsuffix +golang.org/x/net/trace +golang.org/x/net/websocket +# golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d +golang.org/x/oauth2 +golang.org/x/oauth2/clientcredentials +golang.org/x/oauth2/endpoints +golang.org/x/oauth2/google +golang.org/x/oauth2/internal +golang.org/x/oauth2/jws +golang.org/x/oauth2/jwt +# golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +golang.org/x/sync/errgroup +golang.org/x/sync/semaphore +# golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e +golang.org/x/sys/cpu +golang.org/x/sys/unix +golang.org/x/sys/windows +golang.org/x/sys/windows/registry +golang.org/x/sys/windows/svc +golang.org/x/sys/windows/svc/debug +golang.org/x/sys/windows/svc/eventlog +# golang.org/x/text v0.3.2 +golang.org/x/text/cases +golang.org/x/text/collate +golang.org/x/text/encoding +golang.org/x/text/encoding/charmap +golang.org/x/text/encoding/htmlindex +golang.org/x/text/encoding/internal +golang.org/x/text/encoding/internal/identifier +golang.org/x/text/encoding/japanese +golang.org/x/text/encoding/korean +golang.org/x/text/encoding/simplifiedchinese +golang.org/x/text/encoding/traditionalchinese +golang.org/x/text/encoding/unicode +golang.org/x/text/internal +golang.org/x/text/internal/colltab +golang.org/x/text/internal/language +golang.org/x/text/internal/language/compact +golang.org/x/text/internal/tag +golang.org/x/text/internal/utf8internal +golang.org/x/text/language +golang.org/x/text/runes +golang.org/x/text/secure/bidirule +golang.org/x/text/transform +golang.org/x/text/unicode/bidi +golang.org/x/text/unicode/norm +# golang.org/x/time v0.0.0-20191024005414-555d28b269f0 +golang.org/x/time/rate +# golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2 +golang.org/x/tools/cmd/goimports +golang.org/x/tools/cmd/stringer +golang.org/x/tools/go/analysis +golang.org/x/tools/go/analysis/passes/inspect +golang.org/x/tools/go/ast/astutil +golang.org/x/tools/go/ast/inspector +golang.org/x/tools/go/buildutil +golang.org/x/tools/go/gcexportdata +golang.org/x/tools/go/internal/gcimporter +golang.org/x/tools/go/internal/packagesdriver +golang.org/x/tools/go/packages +golang.org/x/tools/go/types/objectpath +golang.org/x/tools/go/types/typeutil +golang.org/x/tools/go/vcs +golang.org/x/tools/internal/fastwalk +golang.org/x/tools/internal/gopathwalk +golang.org/x/tools/internal/imports +golang.org/x/tools/internal/packagesinternal +# golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 +golang.org/x/xerrors +golang.org/x/xerrors/internal +# google.golang.org/api v0.15.0 +google.golang.org/api/cloudfunctions/v1 +google.golang.org/api/compute/v1 +google.golang.org/api/googleapi +google.golang.org/api/googleapi/transport +google.golang.org/api/internal +google.golang.org/api/internal/gensupport +google.golang.org/api/internal/third_party/uritemplates +google.golang.org/api/iterator +google.golang.org/api/option +google.golang.org/api/storage/v1 +google.golang.org/api/support/bundler +google.golang.org/api/transport +google.golang.org/api/transport/grpc +google.golang.org/api/transport/http +google.golang.org/api/transport/http/internal/propagation +# google.golang.org/appengine v1.6.5 +google.golang.org/appengine +google.golang.org/appengine/cloudsql +google.golang.org/appengine/internal +google.golang.org/appengine/internal/app_identity +google.golang.org/appengine/internal/base +google.golang.org/appengine/internal/datastore +google.golang.org/appengine/internal/log +google.golang.org/appengine/internal/modules +google.golang.org/appengine/internal/remote_api +google.golang.org/appengine/internal/socket +google.golang.org/appengine/internal/urlfetch +google.golang.org/appengine/socket +google.golang.org/appengine/urlfetch +# google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb +google.golang.org/genproto/googleapis/api +google.golang.org/genproto/googleapis/api/annotations +google.golang.org/genproto/googleapis/api/distribution +google.golang.org/genproto/googleapis/api/httpbody +google.golang.org/genproto/googleapis/api/label +google.golang.org/genproto/googleapis/api/metric +google.golang.org/genproto/googleapis/api/monitoredres +google.golang.org/genproto/googleapis/datastore/v1 +google.golang.org/genproto/googleapis/iam/v1 +google.golang.org/genproto/googleapis/monitoring/v3 +google.golang.org/genproto/googleapis/pubsub/v1 +google.golang.org/genproto/googleapis/rpc/code +google.golang.org/genproto/googleapis/rpc/status +google.golang.org/genproto/googleapis/type/calendarperiod +google.golang.org/genproto/googleapis/type/expr +google.golang.org/genproto/googleapis/type/latlng +google.golang.org/genproto/protobuf/field_mask +# google.golang.org/grpc v1.29.1 +google.golang.org/grpc +google.golang.org/grpc/attributes +google.golang.org/grpc/backoff +google.golang.org/grpc/balancer +google.golang.org/grpc/balancer/base +google.golang.org/grpc/balancer/grpclb +google.golang.org/grpc/balancer/grpclb/grpc_lb_v1 +google.golang.org/grpc/balancer/roundrobin +google.golang.org/grpc/binarylog/grpc_binarylog_v1 +google.golang.org/grpc/codes +google.golang.org/grpc/connectivity +google.golang.org/grpc/credentials +google.golang.org/grpc/credentials/alts +google.golang.org/grpc/credentials/alts/internal +google.golang.org/grpc/credentials/alts/internal/authinfo +google.golang.org/grpc/credentials/alts/internal/conn +google.golang.org/grpc/credentials/alts/internal/handshaker +google.golang.org/grpc/credentials/alts/internal/handshaker/service +google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp +google.golang.org/grpc/credentials/google +google.golang.org/grpc/credentials/internal +google.golang.org/grpc/credentials/oauth +google.golang.org/grpc/encoding +google.golang.org/grpc/encoding/proto +google.golang.org/grpc/grpclog +google.golang.org/grpc/internal +google.golang.org/grpc/internal/backoff +google.golang.org/grpc/internal/balancerload +google.golang.org/grpc/internal/binarylog +google.golang.org/grpc/internal/buffer +google.golang.org/grpc/internal/channelz +google.golang.org/grpc/internal/envconfig +google.golang.org/grpc/internal/grpclog +google.golang.org/grpc/internal/grpcrand +google.golang.org/grpc/internal/grpcsync +google.golang.org/grpc/internal/grpcutil +google.golang.org/grpc/internal/resolver/dns +google.golang.org/grpc/internal/resolver/passthrough +google.golang.org/grpc/internal/status +google.golang.org/grpc/internal/syscall +google.golang.org/grpc/internal/transport +google.golang.org/grpc/keepalive +google.golang.org/grpc/metadata +google.golang.org/grpc/naming +google.golang.org/grpc/peer +google.golang.org/grpc/resolver +google.golang.org/grpc/serviceconfig +google.golang.org/grpc/stats +google.golang.org/grpc/status +google.golang.org/grpc/tap +# google.golang.org/protobuf v1.23.0 +google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo +google.golang.org/protobuf/compiler/protogen +google.golang.org/protobuf/encoding/protojson +google.golang.org/protobuf/encoding/prototext +google.golang.org/protobuf/encoding/protowire +google.golang.org/protobuf/internal/descfmt +google.golang.org/protobuf/internal/descopts +google.golang.org/protobuf/internal/detectknown +google.golang.org/protobuf/internal/detrand +google.golang.org/protobuf/internal/encoding/defval +google.golang.org/protobuf/internal/encoding/json +google.golang.org/protobuf/internal/encoding/messageset +google.golang.org/protobuf/internal/encoding/tag +google.golang.org/protobuf/internal/encoding/text +google.golang.org/protobuf/internal/errors +google.golang.org/protobuf/internal/fieldnum +google.golang.org/protobuf/internal/fieldsort +google.golang.org/protobuf/internal/filedesc +google.golang.org/protobuf/internal/filetype +google.golang.org/protobuf/internal/flags +google.golang.org/protobuf/internal/genname +google.golang.org/protobuf/internal/impl +google.golang.org/protobuf/internal/mapsort +google.golang.org/protobuf/internal/pragma +google.golang.org/protobuf/internal/set +google.golang.org/protobuf/internal/strs +google.golang.org/protobuf/internal/version +google.golang.org/protobuf/proto +google.golang.org/protobuf/reflect/protodesc +google.golang.org/protobuf/reflect/protoreflect +google.golang.org/protobuf/reflect/protoregistry +google.golang.org/protobuf/runtime/protoiface +google.golang.org/protobuf/runtime/protoimpl +google.golang.org/protobuf/types/descriptorpb +google.golang.org/protobuf/types/known/anypb +google.golang.org/protobuf/types/known/durationpb +google.golang.org/protobuf/types/known/emptypb +google.golang.org/protobuf/types/known/structpb +google.golang.org/protobuf/types/known/timestamppb +google.golang.org/protobuf/types/known/wrapperspb +google.golang.org/protobuf/types/pluginpb +# gopkg.in/inf.v0 v0.9.1 +gopkg.in/inf.v0 +# gopkg.in/jcmturner/aescts.v1 v1.0.1 +gopkg.in/jcmturner/aescts.v1 +# gopkg.in/jcmturner/dnsutils.v1 v1.0.1 +gopkg.in/jcmturner/dnsutils.v1 +# gopkg.in/jcmturner/goidentity.v3 v3.0.0 +gopkg.in/jcmturner/goidentity.v3 +# gopkg.in/jcmturner/gokrb5.v7 v7.5.0 +gopkg.in/jcmturner/gokrb5.v7/asn1tools +gopkg.in/jcmturner/gokrb5.v7/client +gopkg.in/jcmturner/gokrb5.v7/config +gopkg.in/jcmturner/gokrb5.v7/credentials +gopkg.in/jcmturner/gokrb5.v7/crypto +gopkg.in/jcmturner/gokrb5.v7/crypto/common +gopkg.in/jcmturner/gokrb5.v7/crypto/etype +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3961 +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3962 +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc4757 +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc8009 +gopkg.in/jcmturner/gokrb5.v7/gssapi +gopkg.in/jcmturner/gokrb5.v7/iana +gopkg.in/jcmturner/gokrb5.v7/iana/addrtype +gopkg.in/jcmturner/gokrb5.v7/iana/adtype +gopkg.in/jcmturner/gokrb5.v7/iana/asnAppTag +gopkg.in/jcmturner/gokrb5.v7/iana/chksumtype +gopkg.in/jcmturner/gokrb5.v7/iana/errorcode +gopkg.in/jcmturner/gokrb5.v7/iana/etypeID +gopkg.in/jcmturner/gokrb5.v7/iana/flags +gopkg.in/jcmturner/gokrb5.v7/iana/keyusage +gopkg.in/jcmturner/gokrb5.v7/iana/msgtype +gopkg.in/jcmturner/gokrb5.v7/iana/nametype +gopkg.in/jcmturner/gokrb5.v7/iana/patype +gopkg.in/jcmturner/gokrb5.v7/kadmin +gopkg.in/jcmturner/gokrb5.v7/keytab +gopkg.in/jcmturner/gokrb5.v7/krberror +gopkg.in/jcmturner/gokrb5.v7/messages +gopkg.in/jcmturner/gokrb5.v7/pac +gopkg.in/jcmturner/gokrb5.v7/service +gopkg.in/jcmturner/gokrb5.v7/spnego +gopkg.in/jcmturner/gokrb5.v7/types +# gopkg.in/jcmturner/rpc.v1 v1.1.0 +gopkg.in/jcmturner/rpc.v1/mstypes +gopkg.in/jcmturner/rpc.v1/ndr +# gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 +gopkg.in/mgo.v2 +gopkg.in/mgo.v2/bson +gopkg.in/mgo.v2/internal/json +gopkg.in/mgo.v2/internal/sasl +gopkg.in/mgo.v2/internal/scram +# gopkg.in/yaml.v2 v2.3.0 +gopkg.in/yaml.v2 +# honnef.co/go/tools v0.0.1-2019.2.3 +honnef.co/go/tools/arg +honnef.co/go/tools/cmd/staticcheck +honnef.co/go/tools/config +honnef.co/go/tools/deprecated +honnef.co/go/tools/facts +honnef.co/go/tools/functions +honnef.co/go/tools/go/types/typeutil +honnef.co/go/tools/internal/cache +honnef.co/go/tools/internal/passes/buildssa +honnef.co/go/tools/internal/renameio +honnef.co/go/tools/internal/sharedcheck +honnef.co/go/tools/lint +honnef.co/go/tools/lint/lintdsl +honnef.co/go/tools/lint/lintutil +honnef.co/go/tools/lint/lintutil/format +honnef.co/go/tools/loader +honnef.co/go/tools/printf +honnef.co/go/tools/simple +honnef.co/go/tools/ssa +honnef.co/go/tools/ssautil +honnef.co/go/tools/staticcheck +honnef.co/go/tools/staticcheck/vrp +honnef.co/go/tools/stylecheck +honnef.co/go/tools/unused +honnef.co/go/tools/version +# howett.net/plist v0.0.0-20181124034731-591f970eefbb +howett.net/plist +# k8s.io/api v0.18.3 +k8s.io/api/admissionregistration/v1 +k8s.io/api/admissionregistration/v1beta1 +k8s.io/api/apps/v1 +k8s.io/api/apps/v1beta1 +k8s.io/api/apps/v1beta2 +k8s.io/api/auditregistration/v1alpha1 +k8s.io/api/authentication/v1 +k8s.io/api/authentication/v1beta1 +k8s.io/api/authorization/v1 +k8s.io/api/authorization/v1beta1 +k8s.io/api/autoscaling/v1 +k8s.io/api/autoscaling/v2beta1 +k8s.io/api/autoscaling/v2beta2 +k8s.io/api/batch/v1 +k8s.io/api/batch/v1beta1 +k8s.io/api/batch/v2alpha1 +k8s.io/api/certificates/v1beta1 +k8s.io/api/coordination/v1 +k8s.io/api/coordination/v1beta1 +k8s.io/api/core/v1 +k8s.io/api/discovery/v1alpha1 +k8s.io/api/discovery/v1beta1 +k8s.io/api/events/v1beta1 +k8s.io/api/extensions/v1beta1 +k8s.io/api/flowcontrol/v1alpha1 +k8s.io/api/networking/v1 +k8s.io/api/networking/v1beta1 +k8s.io/api/node/v1alpha1 +k8s.io/api/node/v1beta1 +k8s.io/api/policy/v1beta1 +k8s.io/api/rbac/v1 +k8s.io/api/rbac/v1alpha1 +k8s.io/api/rbac/v1beta1 +k8s.io/api/scheduling/v1 +k8s.io/api/scheduling/v1alpha1 +k8s.io/api/scheduling/v1beta1 +k8s.io/api/settings/v1alpha1 +k8s.io/api/storage/v1 +k8s.io/api/storage/v1alpha1 +k8s.io/api/storage/v1beta1 +# k8s.io/apimachinery v0.18.3 +k8s.io/apimachinery/pkg/api/errors +k8s.io/apimachinery/pkg/api/meta +k8s.io/apimachinery/pkg/api/resource +k8s.io/apimachinery/pkg/apis/meta/internalversion +k8s.io/apimachinery/pkg/apis/meta/v1 +k8s.io/apimachinery/pkg/apis/meta/v1/unstructured +k8s.io/apimachinery/pkg/apis/meta/v1beta1 +k8s.io/apimachinery/pkg/conversion +k8s.io/apimachinery/pkg/conversion/queryparams +k8s.io/apimachinery/pkg/fields +k8s.io/apimachinery/pkg/labels +k8s.io/apimachinery/pkg/runtime +k8s.io/apimachinery/pkg/runtime/schema +k8s.io/apimachinery/pkg/runtime/serializer +k8s.io/apimachinery/pkg/runtime/serializer/json +k8s.io/apimachinery/pkg/runtime/serializer/protobuf +k8s.io/apimachinery/pkg/runtime/serializer/recognizer +k8s.io/apimachinery/pkg/runtime/serializer/streaming +k8s.io/apimachinery/pkg/runtime/serializer/versioning +k8s.io/apimachinery/pkg/selection +k8s.io/apimachinery/pkg/types +k8s.io/apimachinery/pkg/util/cache +k8s.io/apimachinery/pkg/util/clock +k8s.io/apimachinery/pkg/util/diff +k8s.io/apimachinery/pkg/util/errors +k8s.io/apimachinery/pkg/util/framer +k8s.io/apimachinery/pkg/util/httpstream +k8s.io/apimachinery/pkg/util/httpstream/spdy +k8s.io/apimachinery/pkg/util/intstr +k8s.io/apimachinery/pkg/util/json +k8s.io/apimachinery/pkg/util/mergepatch +k8s.io/apimachinery/pkg/util/naming +k8s.io/apimachinery/pkg/util/net +k8s.io/apimachinery/pkg/util/runtime +k8s.io/apimachinery/pkg/util/sets +k8s.io/apimachinery/pkg/util/strategicpatch +k8s.io/apimachinery/pkg/util/validation +k8s.io/apimachinery/pkg/util/validation/field +k8s.io/apimachinery/pkg/util/wait +k8s.io/apimachinery/pkg/util/yaml +k8s.io/apimachinery/pkg/version +k8s.io/apimachinery/pkg/watch +k8s.io/apimachinery/third_party/forked/golang/json +k8s.io/apimachinery/third_party/forked/golang/netutil +k8s.io/apimachinery/third_party/forked/golang/reflect +# k8s.io/client-go v0.18.3 +k8s.io/client-go/discovery +k8s.io/client-go/discovery/fake +k8s.io/client-go/kubernetes +k8s.io/client-go/kubernetes/fake +k8s.io/client-go/kubernetes/scheme +k8s.io/client-go/kubernetes/typed/admissionregistration/v1 +k8s.io/client-go/kubernetes/typed/admissionregistration/v1/fake +k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 +k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/fake +k8s.io/client-go/kubernetes/typed/apps/v1 +k8s.io/client-go/kubernetes/typed/apps/v1/fake +k8s.io/client-go/kubernetes/typed/apps/v1beta1 +k8s.io/client-go/kubernetes/typed/apps/v1beta1/fake +k8s.io/client-go/kubernetes/typed/apps/v1beta2 +k8s.io/client-go/kubernetes/typed/apps/v1beta2/fake +k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1 +k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/authentication/v1 +k8s.io/client-go/kubernetes/typed/authentication/v1/fake +k8s.io/client-go/kubernetes/typed/authentication/v1beta1 +k8s.io/client-go/kubernetes/typed/authentication/v1beta1/fake +k8s.io/client-go/kubernetes/typed/authorization/v1 +k8s.io/client-go/kubernetes/typed/authorization/v1/fake +k8s.io/client-go/kubernetes/typed/authorization/v1beta1 +k8s.io/client-go/kubernetes/typed/authorization/v1beta1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v1 +k8s.io/client-go/kubernetes/typed/autoscaling/v1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1 +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2 +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/fake +k8s.io/client-go/kubernetes/typed/batch/v1 +k8s.io/client-go/kubernetes/typed/batch/v1/fake +k8s.io/client-go/kubernetes/typed/batch/v1beta1 +k8s.io/client-go/kubernetes/typed/batch/v1beta1/fake +k8s.io/client-go/kubernetes/typed/batch/v2alpha1 +k8s.io/client-go/kubernetes/typed/batch/v2alpha1/fake +k8s.io/client-go/kubernetes/typed/certificates/v1beta1 +k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake +k8s.io/client-go/kubernetes/typed/coordination/v1 +k8s.io/client-go/kubernetes/typed/coordination/v1/fake +k8s.io/client-go/kubernetes/typed/coordination/v1beta1 +k8s.io/client-go/kubernetes/typed/coordination/v1beta1/fake +k8s.io/client-go/kubernetes/typed/core/v1 +k8s.io/client-go/kubernetes/typed/core/v1/fake +k8s.io/client-go/kubernetes/typed/discovery/v1alpha1 +k8s.io/client-go/kubernetes/typed/discovery/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/discovery/v1beta1 +k8s.io/client-go/kubernetes/typed/discovery/v1beta1/fake +k8s.io/client-go/kubernetes/typed/events/v1beta1 +k8s.io/client-go/kubernetes/typed/events/v1beta1/fake +k8s.io/client-go/kubernetes/typed/extensions/v1beta1 +k8s.io/client-go/kubernetes/typed/extensions/v1beta1/fake +k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1 +k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/networking/v1 +k8s.io/client-go/kubernetes/typed/networking/v1/fake +k8s.io/client-go/kubernetes/typed/networking/v1beta1 +k8s.io/client-go/kubernetes/typed/networking/v1beta1/fake +k8s.io/client-go/kubernetes/typed/node/v1alpha1 +k8s.io/client-go/kubernetes/typed/node/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/node/v1beta1 +k8s.io/client-go/kubernetes/typed/node/v1beta1/fake +k8s.io/client-go/kubernetes/typed/policy/v1beta1 +k8s.io/client-go/kubernetes/typed/policy/v1beta1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1 +k8s.io/client-go/kubernetes/typed/rbac/v1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 +k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1beta1 +k8s.io/client-go/kubernetes/typed/rbac/v1beta1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1 +k8s.io/client-go/kubernetes/typed/scheduling/v1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 +k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 +k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/fake +k8s.io/client-go/kubernetes/typed/settings/v1alpha1 +k8s.io/client-go/kubernetes/typed/settings/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/storage/v1 +k8s.io/client-go/kubernetes/typed/storage/v1/fake +k8s.io/client-go/kubernetes/typed/storage/v1alpha1 +k8s.io/client-go/kubernetes/typed/storage/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/storage/v1beta1 +k8s.io/client-go/kubernetes/typed/storage/v1beta1/fake +k8s.io/client-go/pkg/apis/clientauthentication +k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 +k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 +k8s.io/client-go/pkg/version +k8s.io/client-go/plugin/pkg/client/auth/exec +k8s.io/client-go/rest +k8s.io/client-go/rest/watch +k8s.io/client-go/testing +k8s.io/client-go/tools/auth +k8s.io/client-go/tools/cache +k8s.io/client-go/tools/clientcmd +k8s.io/client-go/tools/clientcmd/api +k8s.io/client-go/tools/clientcmd/api/latest +k8s.io/client-go/tools/clientcmd/api/v1 +k8s.io/client-go/tools/metrics +k8s.io/client-go/tools/pager +k8s.io/client-go/tools/portforward +k8s.io/client-go/tools/reference +k8s.io/client-go/tools/watch +k8s.io/client-go/transport +k8s.io/client-go/transport/spdy +k8s.io/client-go/util/cert +k8s.io/client-go/util/connrotation +k8s.io/client-go/util/flowcontrol +k8s.io/client-go/util/homedir +k8s.io/client-go/util/keyutil +k8s.io/client-go/util/workqueue +# k8s.io/klog v1.0.0 +k8s.io/klog +# k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 +k8s.io/kube-openapi/pkg/util/proto +# k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 +k8s.io/utils/buffer +k8s.io/utils/integer +k8s.io/utils/trace +# sigs.k8s.io/structured-merge-diff/v3 v3.0.0 +sigs.k8s.io/structured-merge-diff/v3/value +# sigs.k8s.io/yaml v1.2.0 +sigs.k8s.io/yaml diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl index 03dcb39c730..81dbde65419 100644 --- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl @@ -124,6 +124,75 @@ inputs: # # Default is false # exponential: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl index 6fd2e4f4e24..59ea36c70bc 100644 --- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl @@ -124,6 +124,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl index 962484e11f2..212d6944b6e 100644 --- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl @@ -124,6 +124,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/_meta/elastic-agent.yml b/x-pack/elastic-agent/_meta/elastic-agent.yml index b7db83d12bd..e8d47c9ea75 100644 --- a/x-pack/elastic-agent/_meta/elastic-agent.yml +++ b/x-pack/elastic-agent/_meta/elastic-agent.yml @@ -119,6 +119,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml index 47a22e9d1b4..261862e7e7f 100644 --- a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml +++ b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml @@ -30,15 +30,87 @@ settings.monitoring: # enables metrics monitoring metrics: true + +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false + inputs: - type: logfile name: epm/nginx version: 1.7.0 dataset.namespace: prod + # constraints are still Experimental and should not be used in production. constraints?: # Contraints look are not final - os.platform: { in: "windows" } @@ -58,6 +130,7 @@ inputs: name: epm/nginx version: 1.7.0 dataset.namespace: prod + # constraints are still Experimental and should not be used in production. constraints?: # Contraints look are not final - os.platform: { in: "windows" } diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml index 962484e11f2..8c51ecd6120 100644 --- a/x-pack/elastic-agent/elastic-agent.docker.yml +++ b/x-pack/elastic-agent/elastic-agent.docker.yml @@ -124,6 +124,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, Elastic-Agent periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/elastic-agent + + # The name of the files where the logs are written to. + #name: elastic-agent + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index 5530d0b455b..d5a5966dfb4 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -130,7 +130,76 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, Elastic-Agent periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/elastic-agent + + # The name of the files where the logs are written to. + #name: elastic-agent + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index a24a07a982d..d1c7c6220be 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -130,7 +130,76 @@ inputs: # # Default is false # exponential: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, Elastic-Agent periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/elastic-agent + + # The name of the files where the logs are written to. + #name: elastic-agent + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go b/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go index 998b90cdfe3..6aa8eda8caa 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go @@ -33,7 +33,7 @@ type actionDispatcher struct { func newActionDispatcher(ctx context.Context, log *logger.Logger, def actionHandler) (*actionDispatcher, error) { var err error if log == nil { - log, err = logger.New() + log, err = logger.New("action_dispatcher") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go index 02e62efd1a4..4205deda8b6 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go @@ -19,7 +19,7 @@ import ( ) func TestActionStore(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("action_store") withFile := func(fn func(t *testing.T, file string)) func(*testing.T) { return func(t *testing.T) { dir, err := ioutil.TempDir("", "action-store") diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go index 4c061abd835..747e5ba5cdc 100644 --- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go @@ -43,7 +43,7 @@ func (m *mockStore) Save(in io.Reader) error { } func TestEnroll(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("tst") t.Run("fail to save is propagated", withTLSServer( func(t *testing.T) *http.ServeMux { diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go index c891cefb056..d9abd6c7aa3 100644 --- a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go +++ b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go @@ -28,6 +28,7 @@ var ( ) // ConstraintFilter filters ast based on included constraints. +// constraints are still Experimental and should not be used in production. func ConstraintFilter(log *logger.Logger, ast *transpiler.AST) error { // get datasources inputsNode, found := transpiler.Lookup(ast, inputsKey) diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go index 64d3f51f3e3..624824e14ec 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go @@ -23,7 +23,7 @@ func TestAcker(t *testing.T) { Events []fleetapi.AckEvent `json:"events"` } - log, _ := logger.New() + log, _ := logger.New("fleet_acker") client := newTestingClient() agentInfo := &testAgentInfo{} acker, err := newActionAcker(log, agentInfo, client) diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go index 4bc48b8fd98..5e1fbe9418e 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go @@ -111,7 +111,7 @@ func withGateway(agentInfo agentInfo, settings *fleetGatewaySettings, fn withGat client := newTestingClient() dispatcher := newTestingDispatcher() - log, _ := logger.New() + log, _ := logger.New("fleet_gateway") rep := getReporter(agentInfo, log, t) ctx, cancel := context.WithCancel(context.Background()) @@ -243,7 +243,7 @@ func TestFleetGateway(t *testing.T) { dispatcher := newTestingDispatcher() ctx, cancel := context.WithCancel(context.Background()) - log, _ := logger.New() + log, _ := logger.New("tst") gateway, err := newFleetGatewayWithScheduler( ctx, log, @@ -328,7 +328,7 @@ func TestFleetGateway(t *testing.T) { dispatcher := newTestingDispatcher() ctx, cancel := context.WithCancel(context.Background()) - log, _ := logger.New() + log, _ := logger.New("tst") gateway, err := newFleetGatewayWithScheduler( ctx, log, diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go index 400052f66f4..b95c259e7c7 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go @@ -29,7 +29,7 @@ func (m *mockEmitter) Emitter(policy *config.Config) error { } func TestPolicyChange(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") ack := newNoopAcker() t.Run("Receive a config change and successfully emits a raw configuration", func(t *testing.T) { @@ -68,7 +68,7 @@ func TestPolicyChange(t *testing.T) { } func TestPolicyAcked(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") t.Run("Config change should not ACK on error", func(t *testing.T) { tacker := &testAcker{} diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go index 5c7ee622a75..8c48384e259 100644 --- a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go @@ -7,11 +7,7 @@ package application import ( "fmt" - "github.com/urso/ecslog" - "github.com/urso/ecslog/backend" - "github.com/urso/ecslog/backend/appender" - "github.com/urso/ecslog/backend/layout" - + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" @@ -202,9 +198,5 @@ func (r *inmemRouter) Dispatch(id string, grpProg map[routingKey][]program.Progr } func newErrorLogger() (*logger.Logger, error) { - backend, err := appender.Console(backend.Error, layout.Text(true)) - if err != nil { - return nil, err - } - return ecslog.New(backend), nil + return logger.NewWithLogpLevel("", logp.ErrorLevel) } diff --git a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go index cd941e64f83..aa48e9bd949 100644 --- a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go @@ -24,7 +24,7 @@ func TestLazyAcker(t *testing.T) { Events []fleetapi.AckEvent `json:"events"` } - log, _ := logger.New() + log, _ := logger.New("") client := newTestingClient() agentInfo := &testAgentInfo{} acker, err := newActionAcker(log, agentInfo, client) diff --git a/x-pack/elastic-agent/pkg/agent/application/local_mode.go b/x-pack/elastic-agent/pkg/agent/application/local_mode.go index dc7a81e198a..b6c8a462018 100644 --- a/x-pack/elastic-agent/pkg/agent/application/local_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/local_mode.go @@ -58,7 +58,7 @@ func newLocal( ) (*Local, error) { var err error if log == nil { - log, err = logger.NewFromConfig(rawConfig) + log, err = logger.NewFromConfig("", rawConfig) if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go index b5303d0dc60..8113a58903d 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go @@ -25,7 +25,7 @@ func TestManagedModeRouting(t *testing.T) { return m, nil } - log, _ := logger.New() + log, _ := logger.New("") router, _ := newRouter(log, streamFn) emit := emitter(log, router, &configModifiers{Decorators: []decoratorFunc{injectMonitoring}, Filters: []filterFunc{filters.ConstraintFilter}}) diff --git a/x-pack/elastic-agent/pkg/agent/application/router.go b/x-pack/elastic-agent/pkg/agent/application/router.go index b48156d7d48..16f529bfa2d 100644 --- a/x-pack/elastic-agent/pkg/agent/application/router.go +++ b/x-pack/elastic-agent/pkg/agent/application/router.go @@ -34,7 +34,7 @@ type router struct { func newRouter(log *logger.Logger, factory streamFunc) (*router, error) { var err error if log == nil { - log, err = logger.New() + log, err = logger.New("router") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index abc5efb3b90..8d01592c8fc 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -67,7 +67,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args } } - logger, err := logger.NewFromConfig(config) + logger, err := logger.NewFromConfig("", config) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index cb0fd62923a..4a53d2fc6ad 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -43,7 +43,7 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { errors.M(errors.MetaKeyPath, pathConfigFile)) } - logger, err := logger.NewFromConfig(config) + logger, err := logger.NewFromConfig("", config) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index 1f323284405..b746d6cd0e2 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -96,7 +96,7 @@ func getLogger() *logger.Logger { "level": "error", }, }) - l, _ := logger.NewFromConfig(cfg) + l, _ := logger.NewFromConfig("", cfg) return l } diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index b92785a11b3..c9c4f9eb9d0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -430,7 +430,7 @@ func TestConfiguration(t *testing.T) { }, } - l, _ := logger.New() + l, _ := logger.New("") for name, test := range testcases { t.Run(name, func(t *testing.T) { singleConfig, err := ioutil.ReadFile(filepath.Join("testdata", name+".yml")) diff --git a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go index df6335f0668..545832c0895 100644 --- a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go +++ b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go @@ -25,7 +25,7 @@ func TestStateResolverAcking(t *testing.T) { } t.Run("when we ACK the should state", func(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") r, err := NewStateResolver(log) require.NoError(t, err) @@ -44,7 +44,7 @@ func TestStateResolverAcking(t *testing.T) { }) t.Run("when we don't ACK the should state", func(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") r, err := NewStateResolver(log) require.NoError(t, err) diff --git a/x-pack/elastic-agent/pkg/agent/warn/warn.go b/x-pack/elastic-agent/pkg/agent/warn/warn.go index c3b97079aa1..03d746992f6 100644 --- a/x-pack/elastic-agent/pkg/agent/warn/warn.go +++ b/x-pack/elastic-agent/pkg/agent/warn/warn.go @@ -11,7 +11,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -const message = "The Elastic Agent is currently in Experimental and should not be used in production" +const message = "The Elastic Agent is currently in BETA and should not be used in production" // LogNotGA warns the users in the log that the Elastic Agent is not GA. func LogNotGA(log *logger.Logger) { diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go index 5c7e2aa9527..cf60bd5e4dd 100644 --- a/x-pack/elastic-agent/pkg/core/logger/logger.go +++ b/x-pack/elastic-agent/pkg/core/logger/logger.go @@ -6,90 +6,103 @@ package logger import ( "fmt" + "path/filepath" - "github.com/urso/ecslog" - "github.com/urso/ecslog/backend" - "github.com/urso/ecslog/backend/appender" - "github.com/urso/ecslog/backend/layout" + "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/logp/configure" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" ) +const agentName = "elastic-agent" + // Logger alias ecslog.Logger with Logger. -type Logger = ecslog.Logger +type Logger = logp.Logger -// Config is a configuration of logging. -type Config struct { - Level loggingLevel `config:"level"` -} +// Config is a logging config. +type Config = logp.Config -// DefaultLoggingConfig creates a default logging configuration. -func DefaultLoggingConfig() *Config { - return &Config{ - Level: loggingLevel(backend.Trace), +// New returns a configured ECS Logger +func New(name string) (*Logger, error) { + dc, err := defaultConfig() + if err != nil { + return nil, err } + return new(name, dc) } -// New returns a configured ECS Logger -func New() (*Logger, error) { - return new(backend.Trace) -} +// NewWithLogpLevel returns a configured logp Logger with specified level. +func NewWithLogpLevel(name string, level logp.Level) (*Logger, error) { + cfg := struct { + Level string `config:"level"` + }{Level: level.String()} -func createJSONBackend(lvl backend.Level) (backend.Backend, error) { - return appender.Console(lvl, layout.Text(true)) + commonCfg, err := common.NewConfigFrom(cfg) + if err != nil { + return nil, err + } + + return new(name, commonCfg) } //NewFromConfig takes the user configuration and generate the right logger. // TODO: Finish implementation, need support on the library that we use. -func NewFromConfig(cfg *config.Config) (*Logger, error) { - wrappedConfig := &struct { - Logging *Config `config:"logging"` - }{ - Logging: DefaultLoggingConfig(), +func NewFromConfig(name string, cfg *config.Config) (*Logger, error) { + defaultCfg, err := defaultConfig() + if err != nil { + return nil, err } + wrappedConfig := &struct { + Logging *common.Config `config:"logging"` + }{Logging: defaultCfg} if err := cfg.Unpack(&wrappedConfig); err != nil { return nil, err } - return new(backend.Level(wrappedConfig.Logging.Level)) + return new(name, wrappedConfig.Logging) } -func new(lvl backend.Level) (*Logger, error) { - backend, err := createJSONBackend(lvl) - if err != nil { - return nil, err +func new(name string, cfg *common.Config) (*Logger, error) { + if err := configure.Logging("", cfg); err != nil { + return nil, fmt.Errorf("error initializing logging: %v", err) } - return ecslog.New(backend), nil -} - -type loggingLevel backend.Level -var loggingLevelMap = map[string]loggingLevel{ - "trace": loggingLevel(backend.Trace), - "debug": loggingLevel(backend.Debug), - "info": loggingLevel(backend.Info), - "error": loggingLevel(backend.Error), + return logp.NewLogger(name), nil } -func (m *loggingLevel) Unpack(v string) error { - mgt, ok := loggingLevelMap[v] - if !ok { - return fmt.Errorf( - "unknown logging level mode, received '%s' and valid values are 'trace', 'debug', 'info' or 'error'", - v, - ) +func defaultConfig() (*common.Config, error) { + cfg := DefaultLoggingConfig() + + // work around custom types and common config + // when custom type is transformed to common.Config + // value is determined based on reflect value which is incorrect + // enum vs human readable form + yamlCfg, err := yaml.Marshal(cfg) + if err != nil { + return nil, err } - *m = mgt - return nil -} -func (m *loggingLevel) String() string { - for s, v := range loggingLevelMap { - if v == *m { - return s - } + commonLogp, err := common.NewConfigFrom(string(yamlCfg)) + if err != nil { + return nil, errors.New(err, errors.TypeConfig) } - return "unknown" + return commonLogp, nil +} + +// DefaultLoggingConfig returns default configuration for agent logging. +func DefaultLoggingConfig() *Config { + cfg := logp.DefaultConfig(logp.DefaultEnvironment) + cfg.Beat = agentName + cfg.ECSEnabled = true + cfg.Level = logp.DebugLevel + cfg.Files.Path = filepath.Join(paths.Home(), "data", "logs") + cfg.Files.Name = agentName + + return &cfg } diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/start.go b/x-pack/elastic-agent/pkg/core/plugin/process/start.go index ee45c4a8abe..9ecaf7d5039 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/start.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/start.go @@ -136,8 +136,6 @@ func injectLogLevel(logLevel string, args []string) []string { var level string // Translate to level beat understands switch logLevel { - case "trace": - level = "debug" case "info": level = "info" case "debug": diff --git a/x-pack/elastic-agent/pkg/core/server/server_test.go b/x-pack/elastic-agent/pkg/core/server/server_test.go index feef37c9bba..9bb82f303ae 100644 --- a/x-pack/elastic-agent/pkg/core/server/server_test.go +++ b/x-pack/elastic-agent/pkg/core/server/server_test.go @@ -578,7 +578,7 @@ func newErrorLogger(t *testing.T) *logger.Logger { }, }) require.NoError(t, err) - log, err := logger.NewFromConfig(cfg) + log, err := logger.NewFromConfig("", cfg) require.NoError(t, err) return log } diff --git a/x-pack/elastic-agent/pkg/filewatcher/watcher.go b/x-pack/elastic-agent/pkg/filewatcher/watcher.go index 32d96436b88..03619a04135 100644 --- a/x-pack/elastic-agent/pkg/filewatcher/watcher.go +++ b/x-pack/elastic-agent/pkg/filewatcher/watcher.go @@ -50,7 +50,7 @@ type Watch struct { func New(log *logger.Logger, f Comparer) (*Watch, error) { var err error if log == nil { - log, err = logger.New() + log, err = logger.New("watcher") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/fleetapi/client.go b/x-pack/elastic-agent/pkg/fleetapi/client.go index bfdc131ce90..c78b0d8ee13 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/client.go +++ b/x-pack/elastic-agent/pkg/fleetapi/client.go @@ -47,7 +47,7 @@ func init() { return nil, err } - l, err := logger.New() + l, err := logger.New("fleet_client") if err != nil { return nil, errors.New(err, "could not create the logger for debugging HTTP request") } diff --git a/x-pack/elastic-agent/pkg/fleetapi/helper_test.go b/x-pack/elastic-agent/pkg/fleetapi/helper_test.go index e9b2baed178..85f66e747ca 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/helper_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/helper_test.go @@ -45,7 +45,7 @@ func withServerWithAuthClient( ) func(t *testing.T) { return withServer(m, func(t *testing.T, host string) { - log, _ := logger.New() + log, _ := logger.New("") cfg := &kibana.Config{ Host: host, } diff --git a/x-pack/elastic-agent/pkg/kibana/client.go b/x-pack/elastic-agent/pkg/kibana/client.go index 33883d2c597..5a20d1b077a 100644 --- a/x-pack/elastic-agent/pkg/kibana/client.go +++ b/x-pack/elastic-agent/pkg/kibana/client.go @@ -95,7 +95,7 @@ func NewConfigFromURL(kURL string) (*Config, error) { func NewWithRawConfig(log *logger.Logger, config *config.Config, wrapper wrapperFunc) (*Client, error) { l := log if l == nil { - log, err := logger.New() + log, err := logger.New("kibana_client") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/kibana/client_test.go b/x-pack/elastic-agent/pkg/kibana/client_test.go index c860d62d88d..818b9a5ea2f 100644 --- a/x-pack/elastic-agent/pkg/kibana/client_test.go +++ b/x-pack/elastic-agent/pkg/kibana/client_test.go @@ -34,7 +34,7 @@ func addCatchAll(mux *http.ServeMux, t *testing.T) *http.ServeMux { } func TestPortDefaults(t *testing.T) { - l, err := logger.New() + l, err := logger.New("") require.NoError(t, err) testCases := []struct { @@ -69,7 +69,7 @@ func TestPortDefaults(t *testing.T) { // - Prefix. func TestHTTPClient(t *testing.T) { ctx := context.Background() - l, err := logger.New() + l, err := logger.New("") require.NoError(t, err) t.Run("Guard against double slashes on path", withServer( diff --git a/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go b/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go index d3a2d902a63..8d25852efe7 100644 --- a/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go +++ b/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go @@ -209,7 +209,7 @@ func getEvents(count int) []reporter.Event { } func newTestReporter(frequency time.Duration, threshold int) *Reporter { - log, _ := logger.New() + log, _ := logger.New("") r := &Reporter{ info: &testInfo{}, queue: make([]fleetapi.SerializableEvent, 0), diff --git a/x-pack/filebeat/docs/inputs/input-awscloudwatch.asciidoc b/x-pack/filebeat/docs/inputs/input-awscloudwatch.asciidoc new file mode 100644 index 00000000000..76f80963d5a --- /dev/null +++ b/x-pack/filebeat/docs/inputs/input-awscloudwatch.asciidoc @@ -0,0 +1,119 @@ +[role="xpack"] + +:libbeat-xpack-dir: ../../../../x-pack/libbeat + +:type: awscloudwatch + +[id="{beatname_lc}-input-{type}"] +=== awscloudwatch input + +++++ +awscloudwatch +++++ + +beta[] + +`awscloudwatch` input can be used to retrieve all logs from all log streams in a +specific log group. `filterLogEvents` AWS API is used to list log events from +the specified log group. Amazon CloudWatch Logs can be used to store log files +from Amazon Elastic Compute Cloud(EC2), AWS CloudTrail, Route53, and other sources. + +A log group is a group of log streams that share the same retention, monitoring, +and access control settings. You can define log groups and specify which streams +to put into each group. There is no limit on the number of log streams that can +belong to one log group. + +A log stream is a sequence of log events that share the same source. Each +separate source of logs in CloudWatch Logs makes up a separate log stream. + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: awscloudwatch + log_group_arn: arn:aws:logs:us-east-1:428152502467:log-group:test:* + scan_frequency: 1m + credential_profile_name: elastic-beats + start_position: beginning +---- + +The `awscloudwatch` input supports the following configuration options plus the +<<{beatname_lc}-input-{type}-common-options>> described later. + +[float] +==== `log_group_arn` +ARN of the log group to collect logs from. + +==== `log_group_name` +Name of the log group to collect logs from. Note: region_name is required when +log_group_name is given. + +==== `region_name` +Region that the specified log group belongs to. + +[float] +==== `log_streams` +A list of strings of log streams names that Filebeat collect log events from. + +[float] +==== `log_stream_prefix` +A string to filter the results to include only log events from log streams +that have names starting with this prefix. + +[float] +==== `start_position` +`start_position` allows user to specify if this input should read log files from +the `beginning` or from the `end`. + +* `beginning`: reads from the beginning of the log group (default). +* `end`: read only new messages from current time minus `scan_frequency` going forward + +For example, with `scan_frequency` equals to `30s` and current timestamp is +`2020-06-24 12:00:00`: + +* with `start_position = beginning`: +** first iteration: startTime=0, endTime=2020-06-24 12:00:00 +** second iteration: startTime=2020-06-24 12:00:00, endTime=2020-06-24 12:00:30 + +* with `start_position = end`: +** first iteration: startTime=2020-06-24 11:59:30, endTime=2020-06-24 12:00:00 +** second iteration: startTime=2020-06-24 12:00:00, endTime=2020-06-24 12:00:30 + +[float] +==== `scan_frequency` +This config parameter sets how often Filebeat checks for new log events from the +specified log group. Default `scan_frequency` is 1 minute, which means Filebeat +will sleep for 1 minute before querying for new logs again. + +[float] +==== `api_timeout` +The maximum duration of AWS API can take. If it exceeds the timeout, AWS API +will be interrupted. The default AWS API timeout for a message is 120 seconds. +The minimum is 0 seconds. The maximum is half of the visibility timeout value. + +[float] +==== `api_sleep` +This is used to sleep between AWS `FilterLogEvents` API calls inside the same +collection period. `FilterLogEvents` API has a quota of 5 transactions per +second (TPS)/account/Region. By default, `api_sleep` is 200 ms. This value should +only be adjusted when there are multiple Filebeats or multiple Filebeat inputs +collecting logs from the same region and AWS account. + +[float] +==== `aws credentials` +In order to make AWS API calls, `awscloudwatch` input requires AWS credentials. +Please see <> for more details. + +[float] +=== AWS Permissions +Specific AWS permissions are required for IAM user to access awscloudwatch: +---- +logs:FilterLogEvents +---- + +[id="{beatname_lc}-input-{type}-common-options"] +include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] + +[id="aws-credentials-config"] +include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] + +:type!: diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index a5719d18fc7..da50998f534 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -8,6 +8,7 @@ package include import ( // Import packages that need to register themselves. + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/googlepubsub" diff --git a/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml b/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml new file mode 100644 index 00000000000..ea5fdd55874 --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml @@ -0,0 +1,20 @@ +- key: awscloudwatch + title: "awscloudwatch" + description: > + Fields from AWS CloudWatch logs. + fields: + - name: awscloudwatch + type: group + default_field: false + description: > + Fields from AWS CloudWatch logs. + fields: + - name: log_group + type: keyword + description: The name of the log group to which this event belongs. + - name: log_stream + type: keyword + description: The name of the log stream to which this event belongs. + - name: ingestion_time + type: keyword + description: The time the event was ingested in AWS CloudWatch. diff --git a/x-pack/filebeat/input/awscloudwatch/config.go b/x-pack/filebeat/input/awscloudwatch/config.go new file mode 100644 index 00000000000..d916ea065da --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/config.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awscloudwatch + +import ( + "errors" + "time" + + "github.com/elastic/beats/v7/filebeat/harvester" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" +) + +type config struct { + harvester.ForwarderConfig `config:",inline"` + LogGroupARN string `config:"log_group_arn"` + LogGroupName string `config:"log_group_name"` + RegionName string `config:"region_name"` + LogStreams []string `config:"log_streams"` + LogStreamPrefix string `config:"log_stream_prefix"` + StartPosition string `config:"start_position" default:"beginning"` + ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"` + APITimeout time.Duration `config:"api_timeout" validate:"min=0,nonzero"` + APISleep time.Duration `config:"api_sleep" validate:"min=0,nonzero"` + AwsConfig awscommon.ConfigAWS `config:",inline"` +} + +func defaultConfig() config { + return config{ + ForwarderConfig: harvester.ForwarderConfig{ + Type: "awscloudwatch", + }, + StartPosition: "beginning", + ScanFrequency: 10 * time.Second, + APITimeout: 120 * time.Second, + APISleep: 200 * time.Millisecond, // FilterLogEvents has a limit of 5 transactions per second (TPS)/account/Region: 1s / 5 = 200 ms + } +} + +func (c *config) Validate() error { + if c.StartPosition != "beginning" && c.StartPosition != "end" { + return errors.New("start_position config parameter can only be " + + "either 'beginning' or 'end'") + } + + if c.LogGroupARN == "" && c.LogGroupName == "" { + return errors.New("log_group_arn and log_group_name config parameter" + + "cannot be both empty") + } + + if c.LogGroupName != "" && c.RegionName == "" { + return errors.New("region_name is required when log_group_name " + + "config parameter is given") + } + return nil +} diff --git a/x-pack/filebeat/input/awscloudwatch/fields.go b/x-pack/filebeat/input/awscloudwatch/fields.go new file mode 100644 index 00000000000..61fa1b5d745 --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package awscloudwatch + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "awscloudwatch", asset.ModuleFieldsPri, AssetAwscloudwatch); err != nil { + panic(err) + } +} + +// AssetAwscloudwatch returns asset data. +// This is the base64 encoded gzipped contents of input/awscloudwatch. +func AssetAwscloudwatch() string { + return "eJys0cFqwzAMBuB7nuKn9/YBfBiMwV5ggx6DFyuxqWMFW5nJ2w87ZU3WyzaqoyL9+oKPuNCioHPqPM8ma+lsA4gTTwqHXf/QAIZSF90kjoPCUwMAr468Segjj3g+v+GlLJzLAjwP6dQAfR1RdfyIoEe6P1lKlokUhsjzdO0Y6vXspa0JCr32ib4/3VF+ySm1JW1Znod2C7ixLrRkjmbT3wneLdUIcA+xVILWP4EwsnWdhViXQJ8UBB/kOdw0+/tJIunxEYA16W8CFwZKJbIVN9J/FGWvEtZLWadrKBm48ONVTs1XAAAA//8tpMbE" +} diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go new file mode 100644 index 00000000000..8c19ca4a24f --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/input.go @@ -0,0 +1,316 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awscloudwatch + +import ( + "context" + "strings" + "sync" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/aws/awserr" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/filebeat/channel" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/logp" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" +) + +const inputName = "awscloudwatch" + +func init() { + err := input.Register(inputName, NewInput) + if err != nil { + panic(errors.Wrapf(err, "failed to register %v input", inputName)) + } +} + +// awsCloudWatchInput is a input for AWS CloudWatch logs +type awsCloudWatchInput struct { + config config + awsConfig awssdk.Config + + logger *logp.Logger + outlet channel.Outleter // Output of received awscloudwatch logs. + inputCtx *channelContext + + workerOnce sync.Once // Guarantees that the worker goroutine is only started once. + workerWg sync.WaitGroup // Waits on awscloudwatch worker goroutine. + stopOnce sync.Once + close chan struct{} + + prevEndTime int64 // track previous endTime for each iteration. +} + +// channelContext implements context.Context by wrapping a channel +type channelContext struct { + done <-chan struct{} +} + +func (c *channelContext) Deadline() (time.Time, bool) { return time.Time{}, false } +func (c *channelContext) Done() <-chan struct{} { return c.done } +func (c *channelContext) Err() error { + select { + case <-c.done: + return context.Canceled + default: + return nil + } +} +func (c *channelContext) Value(key interface{}) interface{} { return nil } + +// NewInput creates a new awscloudwatch input +func NewInput(cfg *common.Config, connector channel.Connector, context input.Context) (input.Input, error) { + cfgwarn.Beta("awsclouwatch input type is used") + logger := logp.NewLogger(inputName) + + // Extract and validate the input's configuration. + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrap(err, "failed unpacking config") + } + logger.Debug("awscloudwatch input config = ", config) + + if config.LogGroupARN != "" { + logGroupName, regionName, err := parseARN(config.LogGroupARN) + if err != nil { + return nil, errors.Wrap(err, "parse log group ARN failed") + } + + config.LogGroupName = logGroupName + config.RegionName = regionName + } + + awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) + if err != nil { + return nil, errors.Wrap(err, "getAWSCredentials failed") + } + awsConfig.Region = config.RegionName + + closeChannel := make(chan struct{}) + in := &awsCloudWatchInput{ + config: config, + awsConfig: awsConfig, + logger: logger, + close: closeChannel, + inputCtx: &channelContext{closeChannel}, + prevEndTime: int64(0), + } + + // Build outlet for events. + in.outlet, err = connector.Connect(cfg) + if err != nil { + return nil, err + } + + in.logger.Info("Initialized AWS CloudWatch input.") + return in, nil +} + +// Run runs the input +func (in *awsCloudWatchInput) Run() { + in.workerOnce.Do(func() { + in.workerWg.Add(1) + go func() { + in.logger.Infof("awscloudwatch input worker for log group: '%v' has started", in.config.LogGroupName) + defer in.logger.Infof("awscloudwatch input worker for log group '%v' has stopped.", in.config.LogGroupName) + defer in.workerWg.Done() + in.run() + }() + }) +} + +func (in *awsCloudWatchInput) run() { + cwConfig := awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "cloudwatchlogs", in.config.RegionName, in.awsConfig) + svc := cloudwatchlogs.New(cwConfig) + for in.inputCtx.Err() == nil { + err := in.getLogEventsFromCloudWatch(svc) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + continue + } + in.logger.Error("getLogEventsFromCloudWatch failed: ", err) + continue + } + + in.logger.Debugf("sleeping for %v before checking new logs", in.config.ScanFrequency) + time.Sleep(in.config.ScanFrequency) + in.logger.Debug("done sleeping") + } +} + +func parseARN(logGroupARN string) (string, string, error) { + arnParsed, err := arn.Parse(logGroupARN) + if err != nil { + return "", "", errors.Errorf("error Parse arn %s: %v", logGroupARN, err) + } + + if strings.Contains(arnParsed.Resource, ":") { + resourceARNSplit := strings.Split(arnParsed.Resource, ":") + if len(resourceARNSplit) >= 2 && resourceARNSplit[0] == "log-group" { + return resourceARNSplit[1], arnParsed.Region, nil + } + } + return "", "", errors.Errorf("cannot get log group name from log group ARN: %s", logGroupARN) +} + +// getLogEventsFromCloudWatch uses FilterLogEvents API to collect logs from CloudWatch +func (in *awsCloudWatchInput) getLogEventsFromCloudWatch(svc cloudwatchlogsiface.ClientAPI) error { + ctx, cancelFn := context.WithTimeout(in.inputCtx, in.config.APITimeout) + defer cancelFn() + + init := true + nextToken := "" + currentTime := time.Now() + startTime, endTime := getStartPosition(in.config.StartPosition, currentTime, in.prevEndTime, in.config.ScanFrequency) + in.logger.Debugf("start_position = %s, startTime = %v, endTime = %v", in.config.StartPosition, time.Unix(startTime/1000, 0), time.Unix(endTime/1000, 0)) + + // overwrite prevEndTime using new endTime + in.prevEndTime = endTime + + for nextToken != "" || init { + // construct FilterLogEventsInput + filterLogEventsInput := in.constructFilterLogEventsInput(startTime, endTime, nextToken) + + // make API request + req := svc.FilterLogEventsRequest(filterLogEventsInput) + resp, err := req.Send(ctx) + if err != nil { + in.logger.Error("failed FilterLogEventsRequest", err) + return err + } + + // get token for next API call, if resp.NextToken is nil, nextToken set to "" + nextToken = "" + if resp.NextToken != nil { + nextToken = *resp.NextToken + } + + logEvents := resp.Events + in.logger.Debugf("Processing #%v events", len(logEvents)) + + err = in.processLogEvents(logEvents) + if err != nil { + err = errors.Wrap(err, "processLogEvents failed") + in.logger.Error(err) + cancelFn() + } + + init = false + + // This sleep is to avoid hitting the FilterLogEvents API limit(5 transactions per second (TPS)/account/Region). + in.logger.Debugf("sleeping for %v before making FilterLogEvents API call again", in.config.APISleep) + time.Sleep(in.config.APISleep) + in.logger.Debug("done sleeping") + } + return nil +} + +func (in *awsCloudWatchInput) constructFilterLogEventsInput(startTime int64, endTime int64, nextToken string) *cloudwatchlogs.FilterLogEventsInput { + filterLogEventsInput := &cloudwatchlogs.FilterLogEventsInput{ + LogGroupName: awssdk.String(in.config.LogGroupName), + StartTime: awssdk.Int64(startTime), + EndTime: awssdk.Int64(endTime), + } + + if len(in.config.LogStreams) > 0 { + filterLogEventsInput.LogStreamNames = in.config.LogStreams + } + + if in.config.LogStreamPrefix != "" { + filterLogEventsInput.LogStreamNamePrefix = awssdk.String(in.config.LogStreamPrefix) + } + + if nextToken != "" { + filterLogEventsInput.NextToken = awssdk.String(nextToken) + } + return filterLogEventsInput +} + +func getStartPosition(startPosition string, currentTime time.Time, prevEndTime int64, scanFrequency time.Duration) (startTime int64, endTime int64) { + switch startPosition { + case "beginning": + if prevEndTime != int64(0) { + return prevEndTime, currentTime.UnixNano() / int64(time.Millisecond) + } + return 0, currentTime.UnixNano() / int64(time.Millisecond) + case "end": + if prevEndTime != int64(0) { + return prevEndTime, currentTime.UnixNano() / int64(time.Millisecond) + } + return currentTime.Add(-scanFrequency).UnixNano() / int64(time.Millisecond), currentTime.UnixNano() / int64(time.Millisecond) + } + return +} + +func (in *awsCloudWatchInput) processLogEvents(logEvents []cloudwatchlogs.FilteredLogEvent) error { + for _, logEvent := range logEvents { + event := createEvent(logEvent, in.config.LogGroupName, in.config.RegionName) + err := in.forwardEvent(event) + if err != nil { + err = errors.Wrap(err, "forwardEvent failed") + in.logger.Error(err) + return err + } + } + return nil +} + +func createEvent(logEvent cloudwatchlogs.FilteredLogEvent, logGroup string, regionName string) beat.Event { + event := beat.Event{ + Timestamp: time.Unix(*logEvent.Timestamp/1000, 0).UTC(), + Fields: common.MapStr{ + "message": *logEvent.Message, + "log.file.path": logGroup + "/" + *logEvent.LogStreamName, + "event": common.MapStr{ + "id": *logEvent.EventId, + "ingested": time.Now(), + }, + "awscloudwatch": common.MapStr{ + "log_group": logGroup, + "log_stream": *logEvent.LogStreamName, + "ingestion_time": time.Unix(*logEvent.IngestionTime/1000, 0), + }, + "cloud": common.MapStr{ + "provider": "aws", + "region": regionName, + }, + }, + } + event.SetID(*logEvent.EventId) + + return event +} + +func (in *awsCloudWatchInput) forwardEvent(event beat.Event) error { + ok := in.outlet.OnEvent(event) + if !ok { + return errors.New("OnEvent returned false. Stopping input worker") + } + return nil +} + +// Stop stops the awscloudwatch input +func (in *awsCloudWatchInput) Stop() { + in.stopOnce.Do(func() { + defer in.outlet.Close() + close(in.close) + in.logger.Info("Stopping awscloudwatch input") + }) +} + +// Wait is an alias for Stop. +func (in *awsCloudWatchInput) Wait() { + in.Stop() + in.workerWg.Wait() +} diff --git a/x-pack/filebeat/input/awscloudwatch/input_test.go b/x-pack/filebeat/input/awscloudwatch/input_test.go new file mode 100644 index 00000000000..6694ad26067 --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/input_test.go @@ -0,0 +1,156 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awscloudwatch + +import ( + "net/http" + "testing" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/cloudwatchlogsiface" + + "github.com/elastic/beats/v7/libbeat/common" + + "github.com/stretchr/testify/assert" +) + +func TestGetStartPosition(t *testing.T) { + currentTime := time.Date(2020, time.June, 1, 0, 0, 0, 0, time.UTC) + cases := []struct { + title string + startPosition string + prevEndTime int64 + scanFrequency time.Duration + expectedStartTime int64 + expectedEndTime int64 + }{ + { + "startPosition=beginning", + "beginning", + int64(0), + 30 * time.Second, + int64(0), + int64(1590969600000), + }, + { + "startPosition=end", + "end", + int64(0), + 30 * time.Second, + int64(1590969570000), + int64(1590969600000), + }, + { + "startPosition=typo", + "typo", + int64(0), + 30 * time.Second, + int64(0), + int64(0), + }, + { + "startPosition=beginning with prevEndTime", + "beginning", + int64(1590000000000), + 30 * time.Second, + int64(1590000000000), + int64(1590969600000), + }, + { + "startPosition=end with prevEndTime", + "end", + int64(1590000000000), + 30 * time.Second, + int64(1590000000000), + int64(1590969600000), + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + startTime, endTime := getStartPosition(c.startPosition, currentTime, c.prevEndTime, c.scanFrequency) + assert.Equal(t, c.expectedStartTime, startTime) + assert.Equal(t, c.expectedEndTime, endTime) + }) + } +} + +// MockCloudwatchlogsClient struct is used for unit tests. +type MockCloudwatchlogsClient struct { + cloudwatchlogsiface.ClientAPI +} + +var ( + mockSvc = &MockCloudwatchlogsClient{} +) + +func (m *MockCloudwatchlogsClient) FilterLogEventsRequest(input *cloudwatchlogs.FilterLogEventsInput) cloudwatchlogs.FilterLogEventsRequest { + events := []cloudwatchlogs.FilteredLogEvent{ + { + EventId: awssdk.String("id-1"), + IngestionTime: awssdk.Int64(1590000000000), + LogStreamName: awssdk.String("logStreamName1"), + Message: awssdk.String("test-message-1"), + Timestamp: awssdk.Int64(1590000000000), + }, + { + EventId: awssdk.String("id-2"), + IngestionTime: awssdk.Int64(1600000000000), + LogStreamName: awssdk.String("logStreamName1"), + Message: awssdk.String("test-message-2"), + Timestamp: awssdk.Int64(1600000000000), + }, + } + + httpReq, _ := http.NewRequest("", "", nil) + return cloudwatchlogs.FilterLogEventsRequest{ + Request: &awssdk.Request{ + Data: &cloudwatchlogs.FilterLogEventsOutput{ + Events: events, + NextToken: awssdk.String(""), + }, + HTTPRequest: httpReq, + }, + } +} + +func TestCreateEvent(t *testing.T) { + logEvent := cloudwatchlogs.FilteredLogEvent{ + EventId: awssdk.String("id-1"), + IngestionTime: awssdk.Int64(1590000000000), + LogStreamName: awssdk.String("logStreamName1"), + Message: awssdk.String("test-message-1"), + Timestamp: awssdk.Int64(1600000000000), + } + + expectedEventFields := common.MapStr{ + "message": "test-message-1", + "event": common.MapStr{ + "id": *logEvent.EventId, + }, + "log.file.path": "logGroup1" + "/" + *logEvent.LogStreamName, + "awscloudwatch": common.MapStr{ + "log_group": "logGroup1", + "log_stream": *logEvent.LogStreamName, + "ingestion_time": time.Unix(*logEvent.IngestionTime/1000, 0), + }, + "cloud": common.MapStr{ + "provider": "aws", + "region": "us-east-1", + }, + } + event := createEvent(logEvent, "logGroup1", "us-east-1") + event.Fields.Delete("event.ingested") + assert.Equal(t, expectedEventFields, event.Fields) +} + +func TestParseARN(t *testing.T) { + logGroup, regionName, err := parseARN("arn:aws:logs:us-east-1:428152502467:log-group:test:*") + assert.Equal(t, "test", logGroup) + assert.Equal(t, "us-east-1", regionName) + assert.NoError(t, err) +} diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index 2fcc2fc8941..63d20221de4 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -7,6 +7,7 @@ package httpjson import ( "regexp" "strings" + "text/template" "time" "github.com/pkg/errors" @@ -35,6 +36,7 @@ type config struct { RetryWaitMax time.Duration `config:"retry.wait_max"` TLS *tlscommon.Config `config:"ssl"` URL string `config:"url" validate:"required"` + DateCursor *DateCursor `config:"date_cursor"` } // Pagination contains information about httpjson pagination settings @@ -65,6 +67,54 @@ type RateLimit struct { Remaining string `config:"remaining"` } +type DateCursor struct { + Enabled *bool `config:"enabled"` + Field string `config:"field" validate:"required"` + URLField string `config:"url_field" validate:"required"` + ValueTemplate *Template `config:"value_template"` + DateFormat string `config:"date_format"` + InitialInterval time.Duration `config:"initial_interval"` +} + +type Template struct { + *template.Template +} + +func (t *Template) Unpack(in string) error { + tpl, err := template.New("tpl").Parse(in) + if err != nil { + return err + } + + *t = Template{Template: tpl} + + return nil +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (dc *DateCursor) IsEnabled() bool { + return dc != nil && (dc.Enabled == nil || *dc.Enabled) +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (dc *DateCursor) GetDateFormat() string { + if dc.DateFormat == "" { + return time.RFC3339 + } + return dc.DateFormat +} + +func (dc *DateCursor) Validate() error { + if dc.DateFormat == "" { + return nil + } + now := time.Now().Format(dc.DateFormat) + if _, err := time.Parse(dc.DateFormat, now); err != nil { + return errors.New("invalid configuration: date_format is not a valid date layout") + } + return nil +} + func (c *config) Validate() error { switch strings.ToUpper(c.HTTPMethod) { case "GET", "POST": @@ -81,6 +131,9 @@ func (c *config) Validate() error { } } if c.Pagination != nil { + if c.DateCursor.IsEnabled() { + return errors.Errorf("invalid configuration: date_cursor cannnot be set in combination with other pagination mechanisms") + } if c.Pagination.Header != nil { if c.Pagination.RequestField != "" || c.Pagination.IDField != "" || len(c.Pagination.ExtraBodyContent) > 0 { return errors.Errorf("invalid configuration: both pagination.header and pagination.req_field or pagination.id_field or pagination.extra_body_content cannot be set simultaneously") diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index cfec6a2440b..a86c2aa76db 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -8,6 +8,7 @@ import ( "context" "os" "testing" + "time" "github.com/pkg/errors" "golang.org/x/oauth2/google" @@ -350,6 +351,32 @@ func TestConfigOauth2Validation(t *testing.T) { "url": "localhost", }, }, + { + name: "date_cursor must fail in combination with pagination", + expectedErr: "invalid configuration: date_cursor cannnot be set in combination with other pagination mechanisms accessing config", + input: map[string]interface{}{ + "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo"}, + "pagination": map[string]interface{}{ + "header": map[string]interface{}{"field_name": "foo", "regex_pattern": "bar"}, + }, + "url": "localhost", + }, + }, + { + name: "date_cursor.date_format will fail if invalid", + expectedErr: "invalid configuration: date_format is not a valid date layout accessing 'date_cursor'", + input: map[string]interface{}{ + "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo", "date_format": "1234"}, + "url": "localhost", + }, + }, + { + name: "date_cursor must work with a valid date_format", + input: map[string]interface{}{ + "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo", "date_format": time.RFC3339}, + "url": "localhost", + }, + }, } for _, c := range cases { diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/httpjson_test.go index 4e70fe72472..75374404eea 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/httpjson_test.go @@ -7,6 +7,7 @@ package httpjson import ( "context" "encoding/json" + "fmt" "io/ioutil" "log" "math/rand" @@ -727,3 +728,50 @@ func TestArrayWithSplitResponse(t *testing.T) { } }) } + +func TestCursor(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "date_cursor.field": "@timestamp", + "date_cursor.url_field": "$filter", + "date_cursor.value_template": "alertCreationTime ge {{.}}", + "date_cursor.initial_interval": "10m", + "date_cursor.date_format": "2006-01-02T15:04:05Z", + } + + timeNow = func() time.Time { + t, _ := time.Parse("2006-01-02T15:04:05Z", "2002-10-02T15:10:00Z") + return t + } + + const ( + expectedQuery = "%24filter=alertCreationTime+ge+2002-10-02T15%3A00%3A00Z" + expectedNextCursorValue = "2002-10-02T15:00:01Z" + expectedNextQuery = "%24filter=alertCreationTime+ge+2002-10-02T15%3A00%3A01Z" + ) + var gotQuery string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + gotQuery = r.URL.Query().Encode() + w.Write([]byte(`[{"@timestamp":"2002-10-02T15:00:00Z"},{"@timestamp":"2002-10-02T15:00:01Z"}]`)) + })) + + runTest(t, ts, m, func(input *HttpjsonInput, out *stubOutleter, t *testing.T) { + group, _ := errgroup.WithContext(context.Background()) + group.Go(input.run) + + events, ok := out.waitForEvents(2) + if !ok { + t.Fatalf("Expected 2 events, but got %d.", len(events)) + } + input.Stop() + + if err := group.Wait(); err != nil { + t.Fatal(err) + } + + assert.Equal(t, expectedQuery, gotQuery) + assert.Equal(t, expectedNextCursorValue, input.nextCursorValue) + assert.Equal(t, fmt.Sprintf("%s?%s", ts.URL, expectedNextQuery), input.getURL()) + }) +} diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 78f5eba5344..fe65f63ebf8 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "net" "net/http" + "net/url" "regexp" "strconv" "sync" @@ -37,6 +38,9 @@ const ( var userAgent = useragent.UserAgent("Filebeat") +// for testing +var timeNow = time.Now + func init() { err := input.Register(inputName, NewInput) if err != nil { @@ -55,6 +59,8 @@ type HttpjsonInput struct { workerCancel context.CancelFunc // Used to signal that the worker should stop. workerOnce sync.Once // Guarantees that the worker goroutine is only started once. workerWg sync.WaitGroup // Waits on worker goroutine. + + nextCursorValue string } // RequestInfo struct has the information for generating an HTTP request @@ -343,6 +349,7 @@ func createRequestInfoFromBody(m common.MapStr, idField string, requestField str // processHTTPRequest processes HTTP request, and handles pagination if enabled func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Client, ri *RequestInfo) error { + ri.URL = in.getURL() for { req, err := in.createHTTPRequest(ctx, ri) if err != nil { @@ -407,8 +414,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl in.log.Debug("http.response.body is not a valid JSON object", string(responseData)) return errors.Errorf("http.response.body is not a valid JSON object, but a %T", obj) } - - if mm != nil && in.config.Pagination != nil && in.config.Pagination.IsEnabled() { + if mm != nil && in.config.Pagination.IsEnabled() { if in.config.Pagination.Header != nil { // Pagination control using HTTP Header url, err := getNextLinkFromHeader(header, in.config.Pagination.Header.FieldName, in.config.Pagination.Header.RegexPattern) @@ -427,7 +433,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl continue } else { // Pagination control using HTTP Body fields - ri, err := createRequestInfoFromBody(common.MapStr(mm), in.config.Pagination.IDField, in.config.Pagination.RequestField, common.MapStr(in.config.Pagination.ExtraBodyContent), in.config.Pagination.URL, ri) + ri, err = createRequestInfoFromBody(common.MapStr(mm), in.config.Pagination.IDField, in.config.Pagination.RequestField, common.MapStr(in.config.Pagination.ExtraBodyContent), in.config.Pagination.URL, ri) if err != nil { return err } @@ -441,10 +447,70 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl continue } } + if mm != nil && in.config.DateCursor.IsEnabled() { + in.advanceCursor(common.MapStr(mm)) + } return nil } } +func (in *HttpjsonInput) getURL() string { + if !in.config.DateCursor.IsEnabled() { + return in.config.URL + } + + var dateStr string + if in.nextCursorValue == "" { + t := timeNow().UTC().Add(-in.config.DateCursor.InitialInterval) + dateStr = t.Format(in.config.DateCursor.GetDateFormat()) + } else { + dateStr = in.nextCursorValue + } + + url, err := url.Parse(in.config.URL) + if err != nil { + return in.config.URL + } + + q := url.Query() + + var value string + if in.config.DateCursor.ValueTemplate == nil { + value = dateStr + } else { + buf := new(bytes.Buffer) + if err := in.config.DateCursor.ValueTemplate.Execute(buf, dateStr); err != nil { + return in.config.URL + } + value = buf.String() + } + + q.Set(in.config.DateCursor.URLField, value) + + url.RawQuery = q.Encode() + + return url.String() +} + +func (in *HttpjsonInput) advanceCursor(m common.MapStr) { + v, err := m.GetValue(in.config.DateCursor.Field) + if err != nil { + in.log.Warnf("date_cursor field: %q", err) + return + } + switch t := v.(type) { + case string: + _, err := time.Parse(in.config.DateCursor.GetDateFormat(), t) + if err != nil { + return + } + in.nextCursorValue = t + default: + in.log.Warn("date_cursor field must be a string, cursor will not advance") + return + } +} + func (in *HttpjsonInput) run() error { ctx, cancel := context.WithCancel(in.workerCtx) defer cancel() @@ -455,7 +521,6 @@ func (in *HttpjsonInput) run() error { } ri := &RequestInfo{ - URL: in.URL, ContentMap: common.MapStr{}, Headers: in.HTTPHeaders, } diff --git a/x-pack/filebeat/module/aws/elb/_meta/fields.yml b/x-pack/filebeat/module/aws/elb/_meta/fields.yml index 9ddfb123901..9499f8bbb0e 100644 --- a/x-pack/filebeat/module/aws/elb/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/elb/_meta/fields.yml @@ -1,6 +1,7 @@ - name: elb type: group release: ga + default_field: false description: > Fields for AWS ELB logs. fields: diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index aee1d393f7b..f1ad3e120c0 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXN9z27aTf89fsdOXrzMj6aZp5+bGN70ZxXGuurqpz1LauycWAlcSzhDAAqAV5a+/WQCkSBGUbEtqvnpIbJFcfHaxv7H0EB5xew1sY98AOOEkXsP4j+kbAIMSmcVrmKNjbwBytNyIwgmtruE/3gAA/KrzUiIstIEVU7kUaglSLy0sjF4TmdEbgIVAmdtr/8AQFFtjtRx93LbAa1gaXRbxm8Q69PnoydSU/TqjeLW5RHMZLnWZO8OErC+lVqTPPrfVJ8cFK6XL/BLXsGDSYutyEmwTsDYe7w1hmRGWFvQU/CYL+ITKZU9orNCqdUfFySNuN9rke9cOAKPPbIVNRJE+6AW4FRLAsDChXzM3SkIrLZpM5KiccNsktH0hd4ENk8iI8iQSBpS4JihcK8eEspCjY0JaYHNdOo+XVgO96NCajH+FCiC4FXOwZjn6Rwz+VaJ1A2Aqh81K8BVwg/5eJi1s0GCHXGkxH8FkAQ7XhTbMbDvP+HsGfoUKt13pjYWV3tC3HZodAnpOXGI+2rs1pSTN3SAZdC4e1pHudiRuCDsSJewZ69nyhnWbfU19OZKuYlRQxmv2VSt4QKtLwxE+sTXC1fjh09sKYGGE4qJgcm/POZNyX6wN1JyjtdkjbjORwncu/GEdIgSTDwHhhlmvOOA0WLFUTQ3tB2zRktFmZBj4xfVCTlnhcwFPFk0sHqgX50a4VcMMLPLSpFQC2ipO5lYbhme9MPpJ5GhBqOBryA3tLDvymKRbi44bZA5z72rdSltsLpl4tM+UmsJdL1jGSrciKpyoJ+8+rhXPFTRE7XhiskQQFpyh/6P4tXbeKYI23qn5nzfEai+xpGeKItptKJNWexm2eA3by9Jip8+vH8eQ45Pg+O+g3QrNRlgchOjYVdimXP1ekdbmzPWBDzI9cMNLBEpkvJN3Yo2wWWGwrq7udiUmrC27jrjNT2WE/l5zkKE+O3wJR+ewR3i9TfbSi+Ht+eGs+hyyRTge3qrPc8wQXiBniNoTY0wMLIeVZgC25KuDJJmFB63dgIz4s0UzIIN+0LLHaJoCqINaOjpdWhBCOTSKSYpZURrNvKoZwZbYryfQ1r3jbKdziUtzO374VHEZNeCKca5LFbbO+1+/d0ZLfHuQXEo8RxTpGVIJYL6NKsTFA2d6o+zltKHiV6gn/Yh5Nk95tHMlZrRUtetUsVk0FOH6CgcydmCp/AKqHPX25h2MS6dhypkvjmMteCuZdYLDe2TKOiYf0wUWGqNNxnW+7/meX/il66smd36ROtGIccWgK42yPjLQ9UP41mgtW54T4uQwmFBeNYjUDqkfaqSVFcywNTo0+/t2qkh3hAckTKa2g2gLFAUtxdYQo/sy+x1SW2hlMYsh+dxAK/p1yKeUkHF6xlaq/ojAV0wt0cJVSK4H3WK4oMzKO8EcJVKSFYi8TfPF8lzQdSYz31rIWavPcipr45o8ZXysUUKERkbtl5R2tFkuKk2HUKV15NyjqA6rVMcHn7pDoQTwgEOpvRBoW4oTUpNowHMUatntKzApMYclKjTM+eeFDaR7jNk3mhJp1kmW3MZfdREaW1IBzBt7ZZBrk/eoUSFO7oYdxTm+n9QtMWat5mJXFfnrGzsuxA2TskPJczAjPg/Ies0UW3rrC7ZwTjuA91pLZKpHjTYrpHqtIW1hYd8QoYEw3NVnAizPtJLp7t/JW7HDKizogvSEdoQA+6WHtPTuQq8/9Rlc2o++pkM5Bims9x817djUwRyE2on2pS28y/XN6mbZ+OFTN2N5VlZ5DhjjmDnuaohKgpRHJsrnszY3j4jGO6faUky1k6EC9oci19dsY4fR7w49smsKNkN61P/eo4FcFIKMvVfApxjMAxYGKcEIvovtZOxt3yBH8eT9q7CHjDnyFTxSFpvsl/Oxdf5Jyw1AKC7LnHLkDaF2RiyXaEJYSDvZ0NQJOlTKPqZWzGAeeTqr2P/z8+RDI3rNt83zFKehVOKvEuW2Uqnm9TRD8XDLC4dKEcrSQ04VvbgNeaTTkIvFAg39Es7q2p+oAjYtkqeCZ6jyQotzi2Rvh3+/v4FqIbKmcMoSc5jYHvJllWe7G4PoeaeBKd9hbBYtdfFVFVrTH9K8cq2slphJvRTpfOE1ASCe7NkCuVgITiBvwkJ3tE7czZc6/+P58WHUXeRpH7xLlG+9GX2gdJmKgOM8HOKjyctaz4XEnjytzck8ZCuntkW7SUOLG0r8vZYFZHAIWcWE15nM6YPwz9P4//xw19mBwwJesKy0Rw4lLibadSmdgAXjjkrH3cEBeapNT0cE4vkp5KUhL59ktXVuv2Gu1U39luf2fxCWF53bH+qKJE7rjhhuG0RFfNQRHPJ3315itzfvwhSIUA3gzxWcKDKW5wbt63OPZFUdGtjoIFLfZaA+HqFJSFPOny/NJXudsO7ev0ivOlnmyYJptj2lZjnMmWSKY08n7aQOQRJAc7agBcAL6ekd3NGX7+OXPVmNY2aJLvPbM+oWUidCbJwHhIWCHuymno629ahkxG6JcyIurz2RcifdR0p/FPL+2rgw2mmu9/PRE0FVVNN7erVyrgBtwPGip1VYN2uNpqpaqGXmxBpHFnkS6UJq9kKH6vVOOybDsbBQYJFrRVmdUBz3pBeagaEvKOxOxqVyQoJopeYMDC5pS6h4mTP+iKqnlRUv/hOx2WCDrkSA4ISUrS+sY8bZ2B+lou1IK/sbc1i3vJt7V9daLS7be+nFI0VvybwTXeBrnY5ZUncatC/jKhpSc6cUrIWUIjI7iNwG+LpAhS2GuNR2/wy+9pzSZuTH7Io94mX5qKZPZndTqJckQXO9LvwZwh5foBNaumIW5ogK0Do2l8Ku+lirzE/s10onerjJ/X4WUSnRTtNDv+WYB64QFtq8vguc9sLauKqHdSo6ctmj+jTEOuZKe9q5ZBJyIAxEuGueV82rXQMND1A8JPt12oRe7J+90P9M94uszLgoVucO1NPpHQS6oQoSiozgX/zX9Sb0pDaE6TKRmpavo/WLcfGVtqgyjsZdNOMK6wCtIxZ+QA1ixzNMGDSU4LXwLRrBzizcQBNUuZ6juTAvQnG99uFV2oxJPLczoeJpiSaeKemF9+B+nUZEnW9TKXxlx4ErfzgeOe4PR/RTnqWaXCfyQbA98ZC/9yAwjOMpndHk0n5KV7k6Yvz5P8Px+qsazmi14ST/E1bI8r6qa00lNOaZKSWlU0InpvtOTtsD1d0m+9yplPUEjIfQHoIRC0p46Z54uefUNiRJ+AV52Z2lPRF47JFXxMOMZyvU7WZJrhbabJjJB7AQXzAfVpFh0BqAHY1Gb0cwccCZqt47AItPaJgM4umxQ4O5MMhdVpoze5PPD3fRQ3uJx3X8ATWvGm+1CA6M64wMMnvuN0nCDFGgXI0S1dsR8S2YkE1kdXT7IZydXrzTMv3BHx+gqSbwX9J2mZf8EV2WOqU81S8wpZXgTIah7t3RqF9rbwoxwOhJ1vy1y/WGAv3gC/ZnkKO10W4vmVD9s01r7TDrScc7Xz/H9ouCmRAtn93eg26r4W/b1Ho9PzjFYOjVs1Tt8fvqjOxig0cd0GOwzvflWyea9ela+kAT2e48Lw21ns04r3h3syC++ZXDKozrQ45cMkr9mYXpb+P7UX3nAB5up7PRz7PZfbZGt9L5qJo98ENPA/jj9v10Mrs9dIs28H48u/l59OH27nZ2O/rt/X/d3szSrD/imePzd4+4/a45uraLwhQcUFFVk3uQ3w2/q9zwTlS5xjD95qjoZv4FpHpq8bCmlUacl5eHQHj4+WHS4ohkXzuWzilHExrVdVko587Yq1DlGo3gAUez4NyNqxyYCDzDvG66FqrN8NYH2hudY3OflY4RWHNeGtPbDdk6tJntmzd7tcRieVN32fw6vkQfAH6pxjq8SHet4Sc0lO822fiKRve4kfn/UVJlxde0ZE9pTBHROuT6dajy8hrY2xfxT/pW2UUk2eqA7YdaoWAhxXLlGtMiPq35h4UCjS0oLXzq0VBXGpUxo0uV/23wmWsosC0oWDeS860uzbGB6AUac+4I3SpyvG4+xHViHXa8UvXvP7PlKfObz4T22aIZjmmlg0VinGo4e+lajcJOPlQNxTryPDfYRBKT/FjIWekzZzbEwJchW38dinz4zr+BUWsjfnGo8l3CBZMPPS04sVTMlaZvpuXUVlFFvpLTAKZi+btHSz/8OOi+V9PMGFtO4tV5ZehPZrYUnRcvT+0zIifmppoqCAt3bIsGrqbTu7dVU3Q3no9L7UT9Oi+p/zTFGl3o6TW0BlJOm2hPx414Xl2/Q9uegAl/AWBcutXP3lbDCxbte4IV2wH8d4lmOw2pN933F/1e5eJXhcEh6QbmlOK9ff3WeqsKi5658q/m+yq1jK1K+vHIbJ6T9jLWNDNMWX/8ERRtWr1idzW7m76tvVlD02Lncv+orzExuZB68/wORWey5rk9it/vb4CWelFv4iJCJCQfCcmdXtpqCf/HL7a6pN2OL2H5maE4JB1ezKrkKyy8qx8Is5JbYMBL6/S674keXTnD4HQ6s/YjtvXAdHU+WW1BX7PdoVlcoke86xModBttHndreWxhgtZPoRi2WAgeT7S1yQ93Xi/ScK2GklNvyER8Axjf3Nzez/xrzrf9xbLUy0PF3KuRSr1ckieNpVwUbrW9A/jtlwF8+u3DeDb2ofaXyT393Lft1jF10V2vlvCi/UdXsq/QikGVm9W0hfWtRe/1trrsmQx6dJk1nOV5OmC8pldXMAr/Q4lPKOFKG7EUism3VW+ze6ge2elHmFv3tyDMqRhUIXQ3YFbu4iDOp4JfUGP8jDzZYf3Xkc7qPWw5V3h+t7vDHxa4JAuOF9lCsuWZPctcuDWzj7FYqwOHllJvyOPMbu7BL3sN736a/u+nwff/Rv8Nxze/DL7/6ePk0+DHnx6mszTky41YBqldw+T+6ccB/fuvvoa7/Tgevfn/AAAA//8RgkyD" + return "eJzMXN9z27aTf89fsdOXrzMj6aZp5+bGN70ZxXGuurqpz1LauycWAlcSzhDAAqAV5a+/WQCkSBGUbEtqvnpIbJFcfHaxv7H0EB5xew1sY98AOOEkXsP4j+kbAIMSmcVrmKNjbwBytNyIwgmtruE/3gAA/KrzUiIstIEVU7kUaglSLy0sjF4TmdEbgIVAmdtr/8AQFFtjtRx93LbAa1gaXRbxm8Q69PnoydSU/TqjeLW5RHMZLnWZO8OErC+lVqTPPrfVJ8cFK6XL/BLXsGDSYutyEmwTsDYe7w1hmRGWFvQU/CYL+ITKZU9orNCqdUfFySNuN9rke9cOAKPPbIVNRJE+6AW4FRLAsDChXzM3SkIrLZpM5KiccNsktH0hd4ENk8iI8iQSBpS4JihcK8eEspCjY0JaYHNdOo+XVgO96NCajH+FCiC4FXOwZjn6Rwz+VaJ1A2Aqh81K8BVwg/5eJi1s0GCHXGkxH8FkAQ7XhTbMbDvP+HsGfoUKt13pjYWV3tC3HZodAnpOXGI+2rs1pSTN3SAZdC4e1pHudiRuCDsSJewZ69nyhnWbfU19OZKuYlRQxmv2VSt4QKtLwxE+sTXC1fjh09sKYGGE4qJgcm/POZNyX6wN1JyjtdkjbjORwncu/GEdIgSTDwHhhlmvOOA0WLFUTQ3tB2zRktFmZBj4xfVCTlnhcwFPFk0sHqgX50a4VcMMLPLSpFQC2ipO5lYbhme9MPpJ5GhBqOBryA3tLDvymKRbi44bZA5z72rdSltsLpl4tM+UmsJdL1jGSrciKpyoJ+8+rhXPFTRE7XhiskQQFpyh/6P4tXbeKYI23qn5nzfEai+xpGeKItptKJNWexm2eA3by9Jip8+vH8eQ45Pg+O+g3QrNRlgchOjYVdimXP1ekdbmzPWBDzI9cMNLBEpkvJN3Yo2wWWGwrq7udiUmrC27jrjNT2WE/l5zkKE+O3wJR+ewR3i9TfbSi+Ht+eGs+hyyRTge3qrPc8wQXiBniNoTY0wMLIeVZgC25KuDJJmFB63dgIz4s0UzIIN+0LLHaJoCqINaOjpdWhBCOTSKSYpZURrNvKoZwZbYryfQ1r3jbKdziUtzO374VHEZNeCKca5LFbbO+1+/d0ZLfHuQXEo8RxTpGVIJYL6NKsTFA2d6o+zltKHiV6gn/Yh5Nk95tHMlZrRUtetUsVk0FOH6CgcydmCp/AKqHPX25h2MS6dhypkvjmMteCuZdYLDe2TKOiYf0wUWGqNNxnW+7/meX/il66smd36ROtGIccWgK42yPjLQ9UP41mgtW54T4uQwmFBeNYjUDqkfaqSVFcywNTo0+/t2qkh3hAckTKa2g2gLFAUtxdYQo/sy+x1SW2hlMYsh+dxAK/p1yKeUkHF6xlaq/ojAV0wt0cJVSK4H3WK4oMzKO8EcJVKSFYi8TfPF8lzQdSYz31rIWavPcipr45o8ZXysUUKERkbtl5R2tFkuKk2HUKV15NyjqA6rVMcHn7pDoQTwgEOpvRBoW4oTUpNowHMUatntKzApMYclKjTM+eeFDaR7jNk3mhJp1kmW3MZfdREaW1IBzBt7ZZBrk/eoUSFO7oYdxTm+n9QtMWat5mJXFfnrGzsuxA2TskPJczAjPg/Ies0UW3rrC7ZwTjuA91pLZKpHjTYrpHqtIW1hYd8QoYEw3NVnAizPtJLp7t/JW7HDKizogvSEdoQA+6WHtPTuQq8/9Rlc2o++pkM5Bims9x817djUwRyE2on2pS28y/XN6mbZ+OFTN2N5VlZ5DhjjmDnuaohKgpRHJsrnszY3j4jGO6faUky1k6EC9oci19dsY4fR7w49smsKNkN61P/eo4FcFIKMvVfApxjMAxYGKcEIvovtZOxt3yBH8eT9q7CHjDnyFTxSFpvsl/Oxdf5Jyw1AKC7LnHLkDaF2RiyXaEJYSDvZ0NQJOlTKPqZWzGAeeTqr2P/z8+RDI3rNt83zFKehVOKvEuW2Uqnm9TRD8XDLC4dKEcrSQ04VvbgNeaTTkIvFAg39Es7q2p+oAjYtkqeCZ6jyQotzi2Rvh3+/v4FqIbKmcMoSc5jYHvJllWe7G4PoeaeBKd9hbBYtdfFVFVrTH9K8cq2slphJvRTpfOE1ASCe7NkCuVgITiBvwkJ3tE7czZc6/+P58WHUXeRpH7xLlG+9GX2gdJmKgOM8HOKjyctaz4XEnjytzck8ZCuntkW7SUOLG0r8vZYFZHAIWcWE15nM6YPwz9P4//xw19mBwwJesKy0Rw4lLibadSmdgAXjjkrH3cEBeapNT0cE4vkp5KUhL59ktXVuv2Gu1U39luf2fxCWF53bH+qKJE7rjhhuG0RFfNQRHPJ3315itzfvwhSIUA3gzxWcKDKW5wbt63OPZFUdGtjoIFLfZaA+HqFJSFPOny/N5WVkeff+RWrXSUJPlluzKyo1y2HOJFMcexptJzUQkgCaowctAF5IT+/gjr58H7/sSXocM0t0md+9UbfOOhFi47ggLBTUZDcUdbTrRxUldiugE3F57YmUO9UAUnakkPeXzoXRTnO9n66eCKqimt7Tq5VzBWgDjhc9ncS6l2s0Fd1CLTMn1jiyyJNIF1KzF/pbr3faMRlOjYUCi1wrSvqE4rgnvdArDG1DYXcyLpUTEkQrc2dgcElbQrXNnPFHVD2drnjxn4jNBht0JQIEJ6RsfWEdM87G9inVdEc63d+Yw7oj3ty7uhRrcdneSy8eKXor6p3oAl/rdEiTutO/fRlX0ZCaO6VgLaQUkdlB5DbA1wUqbDHEpbb7R/S155Q2Iz9mV+wRL8tHNZwyu5tCvSQJmut14Y8Y9vgCndDSFbMwR1SA1rG5FHbVx1plfmK/lDrRw03u95OMSol2mh7aMcc8cIWw0Ob1TeK0F9bGVS2uU9GRyx7VhyXWMVfa044tk5ADYSDCXfO8al7tGmh4gOIh2a/TJrRq/+yF/me6nWRlxkWxOnegnk7vINANRZJQZAT/4r+uN6EntSFMl4nUtHwdrV+Mi6+0RZVxNO6iGVdYB2gdsfDzaxAbomEAoaEEr4Vv0Qh2ZuEGmqDK9RzNhXkRiuu1D6/SZkziuZ0J1VZLNPHISS+8B/frNCLqfJtK4Ss7Dlz5s/PIcX84op/yLNUDO5EPgu2Jh/y9B4FhHE9pnCaX9kO8ytUR48//GY7XX9VwRqsNJ/mfsEKW91Vda6qwMc9MKSmdEjox/Hdy2h6o7jbZ506lrAdkPIT2jIxYUMJL98TLPYe6IUnCL8jL7qjticBjC70iHkZAW6FuN2pytdBmw0w+gIX4gvmwigyD1nzsaDR6O4KJA85U9VoCWHxCw2QQT48dGsyFQe6y0pzZm3x+uIse2ks8ruPPr3nVl6tFcGCaZ2SQ2XO/aBJGjALlatKo3o6Ib8GEbCKro9sP4Wj1Wzdipj/4wwc01fz+S7oy85I/ostSZ5ynug2mtBKcyTASvjtY9WvtzTAGGD25nL92udZRoB9cxf4EczRGUoYlE6p/MmqtHWY92Xrn6+e4hqJgJgTTZzcHoduJ+Ns2tV7Pj10xGHr1LFV7eL86YbvY2FIH9Bis81391nlofTaXPg5FtjsNTEOtJzvOK97dJInvjeWwCsP+kCOXjCoDZmH62/h+VN85gIfb6Wz082x2n63RrXQ+qiYX/MjUAP64fT+dzG4P3aINvB/Pbn4efbi9u53djn57/1+3N7M064945vD93SNuv2sOvu2CNMUOVFT05B7kd8PvKi+9E1WuMczOOarJmX99qZ55PKxppRHn5eUhEB5+fpi0OCLZ146lc0bShEZlXxaqvTO2MlS5RiN4wNGsR3fDLgfmCc8w7ZsulWozvPVx+Ebn2NxnpWOA1pyXxvQ2S7YObWb7ptVeLbFY/dRNOL+Or+AHgF+qoRAv0l3n+AkNpcNNNr6i0T1uZP5/lHNZ8TUt2VP6VkS0Drl+HSrMvAb2tk38k76TdhFJthpk+6FWKFhIsVy5xqyJT2v+YaFAYwvKGp96NNSVRmXM6FLlfxt85hoKbAsK1o3cfatLc2yceoHGnDtCt2ogr5sPcZ1Yph0vZP3b02x5yvTnM6F9tmiGY1rpYA0ZZyLOXtlWg7STD1W/sY48zw02kcQkPxZyVvrMmQ0x8GXI1l+HIh++8+9v1NqIXxyqfJdwweRDT4dOLBVzpembiDm1k1SRr+Q0gKlY/u7R0g8/Drpv5TQzxpaTeHVeGdqXmS1F57XNU9uQyIm5qaYKwsId26KBq+n07m3VM90N9+NSO1G/DEzqP02xRhd6WhGtcZbT5uHTcSMeZ9dv4LbnZ8LfDxiXbvWzt9Xwekb7nmDFdgD/XaLZTkPqTff9Rb9XufhVYXBIuoE5pXhvX7+13qrComduDFTTgZVaxk4m/Xhkss9JexlrmhmmrD8dCYo2rV7Qu5rdTd/W3qyhabGxuX8S2Ji3XEi9eX4D41JzOb/f3wAheVHr4iIyJiQfCcmdXtpqCf+XNba6JGWIb3h5xuMEdnjrqxK/sPCufiAMYm6BAS+t0+u+J3pU6QxT2enE28/v1tPY1elmtQV9rXqHZnGJDvOujaDQbbR53K3lsYXxXD/DYthiIXg8D9cmP9y3PS/MvYnn1Os3Ed8Axjc3t/cz/w71bX8tLfXyUK33aqRSL5fkaGOlF4Vbbe8AfvtlAJ9++zCejX0k/mVyTz/3bbt1TF1016slvGj/0ZXsK7RiUKVuNW1hfefRO8WtLnvmih5dZg1neZ6OJ69p5RWMsoOhxCeUcKWNWArF5Nuq9dk9ko/s9CPMrftbEOZUK6oQ2RswK3dxEOdTwS+oMX4An+yw/tNLZ/UetpwrPL/b3eEPC1ySBceLbCHZ8syeZS7cmtnHWMvVgUNLqTfkcWY39+CXvYZ3P03/99Pg+3+j/4bjm18G3//0cfJp8ONPD9NZGvLlBjSD1K5hcv/044D+/Vdf4t1+HI/e/H8AAAD//6QmZ2g=" } diff --git a/x-pack/filebeat/module/aws/s3access/_meta/fields.yml b/x-pack/filebeat/module/aws/s3access/_meta/fields.yml index 7451a258831..0bb6d8f8326 100644 --- a/x-pack/filebeat/module/aws/s3access/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/s3access/_meta/fields.yml @@ -1,6 +1,7 @@ - name: s3access type: group release: ga + default_field: false description: > Fields for AWS S3 server access logs. fields: diff --git a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml index 1fbd4b37562..22686069848 100644 --- a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml @@ -1,6 +1,7 @@ - name: vpcflow type: group release: beta + default_field: false description: > Fields for AWS VPC flow logs. fields: diff --git a/x-pack/filebeat/module/suricata/eve/_meta/fields.yml b/x-pack/filebeat/module/suricata/eve/_meta/fields.yml index 7529cba1b52..45980b888b0 100644 --- a/x-pack/filebeat/module/suricata/eve/_meta/fields.yml +++ b/x-pack/filebeat/module/suricata/eve/_meta/fields.yml @@ -665,6 +665,24 @@ - name: subject type: keyword + - name: ja3s + type: group + default_field: false + fields: + - name: string + type: keyword + - name: hash + type: keyword + + - name: ja3 + type: group + default_field: false + fields: + - name: string + type: keyword + - name: hash + type: keyword + - name: app_proto_ts type: keyword diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml index 700c8db17e9..438c30e6373 100644 --- a/x-pack/filebeat/module/suricata/eve/config/eve.yml +++ b/x-pack/filebeat/module/suricata/eve/config/eve.yml @@ -373,26 +373,23 @@ processors: addTlsVersion(evt); cleanupTlsSni(evt); } - - if: - equals: - suricata.eve.event_type: tls - then: - - convert: - ignore_missing: true - ignore_failure: true - mode: copy - fields: - - {from: suricata.eve.tls.subject, to: tls.server.subject} - - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer} - - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean} - - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1} - - {from: suricata.eve.tls.sni, to: tls.client.server_name} - - {from: suricata.eve.tls.sni, to: destination.domain} - - {from: suricata.eve.tls.notbefore, to: tls.server.not_before} - - {from: suricata.eve.tls.notafter, to: tls.server.not_after} - - {from: suricata.eve.tls.ja3s, to: tls.server.ja3s} - - {from: suricata.eve.tls.certificate, to: tls.server.certificate} - - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain} + - convert: + ignore_missing: true + fail_on_error: false + mode: copy + fields: + - {from: suricata.eve.tls.subject, to: tls.server.subject} + - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer} + - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean} + - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1} + - {from: suricata.eve.tls.sni, to: tls.client.server_name} + - {from: suricata.eve.tls.sni, to: destination.domain} + - {from: suricata.eve.tls.notbefore, to: tls.server.not_before} + - {from: suricata.eve.tls.notafter, to: tls.server.not_after} + - {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s} + - {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3} + - {from: suricata.eve.tls.certificate, to: tls.server.certificate} + - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain} - drop_fields: ignore_missing: true fields: diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log index 81bc39dbf4f..6587a913f52 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log @@ -18,3 +18,5 @@ {"timestamp":"2018-10-04T09:35:00.897009+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":7,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/binary-amd64\/by-hash\/SHA256\/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":330,"pkts_toclient":591,"bytes_toserver":23758,"bytes_toclient":884342,"start":"2018-10-04T09:34:58.926006+0000"}} {"timestamp":"2018-10-04T09:35:01.362208+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":8,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/i18n\/by-hash\/SHA256\/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":524,"pkts_toclient":979,"bytes_toserver":36819,"bytes_toclient":1467603,"start":"2018-10-04T09:34:58.926006+0000"}} {"timestamp":"2018-10-04T09:35:01.575088+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":9,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/multiverse\/binary-amd64\/by-hash\/SHA256\/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":575,"pkts_toclient":1079,"bytes_toserver":40452,"bytes_toclient":1618380,"start":"2018-10-04T09:34:58.926006+0000"}} +{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN","subject":"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"} +{"flow":{"start":"2020-06-26T11:00:02.970011-0400","bytes_toclient":4660,"bytes_toserver":1074,"pkts_toclient":8,"pkts_toserver":7},"app_proto":"tls","tls":{"ja3s":{"string":"742,48172,30210-30","hash":"391231ba5675e42807b9e1f457b2614e"},"ja3":{"string":"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3","hash":"3f1ea03f5822e8021b60cc3e4b233181"},"notafter":"2026-06-25T17:36:29","notbefore":"2016-06-27T17:36:29","version":"TLS 1.2","sni":"host.domain.net","fingerprint":"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc","serial":"72:A9:2C:51","issuerdn":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown","subject":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown"},"alert":{"severity":3,"category":"","signature":"SURICATA TLS on unusual port","rev":1,"signature_id":2610003,"gid":1,"action":"allowed"},"proto":"TCP","dest_port":8443,"dest_ip":"10.128.2.48","src_port":64389,"src_ip":"10.137.3.54","event_type":"alert","in_iface":"enp0s31f6","flow_id":991192778198299,"timestamp":"2020-06-26T11:00:03.342282-0400"} diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 793ce164746..6e06de5c66e 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -1536,5 +1536,146 @@ "user_agent.original": "Debian APT-HTTP/1.3 (1.6.3ubuntu0.1)", "user_agent.os.name": "Debian", "user_agent.version": "1.3" + }, + { + "@timestamp": "2018-10-04T09:35:02.796Z", + "destination.address": "10.232.0.237", + "destination.domain": "hostname.domain.net", + "destination.ip": "10.232.0.237", + "destination.port": 9080, + "event.category": [ + "network" + ], + "event.dataset": "suricata.eve", + "event.kind": "event", + "event.module": "suricata", + "event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN\",\"subject\":\"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}", + "event.type": [ + "protocol" + ], + "fileset.name": "eve", + "input.type": "log", + "log.offset": 16546, + "network.community_id": "1:qsGDjYDIWp+kHhxotTdhPbUaWSo=", + "network.protocol": "tls", + "network.transport": "tcp", + "related.hash": [ + "00112233445566778899AABBCCDDEEFF00112233" + ], + "related.ip": [ + "10.126.2.140", + "10.232.0.237" + ], + "service.type": "suricata", + "source.address": "10.126.2.140", + "source.ip": "10.126.2.140", + "source.port": 45884, + "suricata.eve.event_type": "tls", + "suricata.eve.flow_id": 1091813059495729, + "suricata.eve.in_iface": "enp5s0", + "suricata.eve.tls.fingerprint": "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33", + "suricata.eve.tls.issuerdn": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN", + "suricata.eve.tls.ja3.hash": "d92325c876e7279f4eb8c62415e3a6b7", + "suricata.eve.tls.ja3.string": "001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0", + "suricata.eve.tls.ja3s.hash": "0993626a07ad09e1ce91293be7aa5721", + "suricata.eve.tls.ja3s.string": "333,55555,66666-22", + "suricata.eve.tls.notafter": "2024-07-16T14:52:35", + "suricata.eve.tls.notbefore": "2019-07-17T14:52:35", + "suricata.eve.tls.serial": "00:11:22:33:44:55:66:77:88", + "suricata.eve.tls.sni": "hostname.domain.net", + "suricata.eve.tls.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com", + "suricata.eve.tls.version": "TLS 1.2", + "tags": [ + "suricata" + ], + "tls.client.ja3": "d92325c876e7279f4eb8c62415e3a6b7", + "tls.client.server_name": "hostname.domain.net", + "tls.server.hash.sha1": "00112233445566778899AABBCCDDEEFF00112233", + "tls.server.issuer": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN", + "tls.server.ja3s": "0993626a07ad09e1ce91293be7aa5721", + "tls.server.not_after": "2024-07-16T14:52:35", + "tls.server.not_before": "2019-07-17T14:52:35", + "tls.server.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com", + "tls.version": "1.2", + "tls.version_protocol": "tls" + }, + { + "@timestamp": "2020-06-26T15:00:03.342Z", + "destination.address": "10.128.2.48", + "destination.bytes": 4660, + "destination.domain": "host.domain.net", + "destination.ip": "10.128.2.48", + "destination.packets": 8, + "destination.port": 8443, + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "suricata.eve", + "event.kind": "alert", + "event.module": "suricata", + "event.original": "{\"flow\":{\"start\":\"2020-06-26T11:00:02.970011-0400\",\"bytes_toclient\":4660,\"bytes_toserver\":1074,\"pkts_toclient\":8,\"pkts_toserver\":7},\"app_proto\":\"tls\",\"tls\":{\"ja3s\":{\"string\":\"742,48172,30210-30\",\"hash\":\"391231ba5675e42807b9e1f457b2614e\"},\"ja3\":{\"string\":\"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3\",\"hash\":\"3f1ea03f5822e8021b60cc3e4b233181\"},\"notafter\":\"2026-06-25T17:36:29\",\"notbefore\":\"2016-06-27T17:36:29\",\"version\":\"TLS 1.2\",\"sni\":\"host.domain.net\",\"fingerprint\":\"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc\",\"serial\":\"72:A9:2C:51\",\"issuerdn\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\",\"subject\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\"},\"alert\":{\"severity\":3,\"category\":\"\",\"signature\":\"SURICATA TLS on unusual port\",\"rev\":1,\"signature_id\":2610003,\"gid\":1,\"action\":\"allowed\"},\"proto\":\"TCP\",\"dest_port\":8443,\"dest_ip\":\"10.128.2.48\",\"src_port\":64389,\"src_ip\":\"10.137.3.54\",\"event_type\":\"alert\",\"in_iface\":\"enp0s31f6\",\"flow_id\":991192778198299,\"timestamp\":\"2020-06-26T11:00:03.342282-0400\"}", + "event.severity": 3, + "event.start": "2020-06-26T15:00:02.970Z", + "event.type": [ + "allowed" + ], + "fileset.name": "eve", + "input.type": "log", + "log.offset": 17606, + "message": "", + "network.bytes": 5734, + "network.community_id": "1:W6fjhboFUwyEchJ3ELaqSBzDEJE=", + "network.packets": 15, + "network.protocol": "tls", + "network.transport": "tcp", + "related.hash": [ + "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC" + ], + "related.ip": [ + "10.137.3.54", + "10.128.2.48" + ], + "rule.id": "2610003", + "rule.name": "SURICATA TLS on unusual port", + "service.type": "suricata", + "source.address": "10.137.3.54", + "source.bytes": 1074, + "source.ip": "10.137.3.54", + "source.packets": 7, + "source.port": 64389, + "suricata.eve.alert.category": "", + "suricata.eve.alert.gid": 1, + "suricata.eve.alert.rev": 1, + "suricata.eve.alert.signature": "SURICATA TLS on unusual port", + "suricata.eve.alert.signature_id": 2610003, + "suricata.eve.event_type": "alert", + "suricata.eve.flow_id": 991192778198299, + "suricata.eve.in_iface": "enp0s31f6", + "suricata.eve.tls.fingerprint": "36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc", + "suricata.eve.tls.issuerdn": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "suricata.eve.tls.ja3.hash": "3f1ea03f5822e8021b60cc3e4b233181", + "suricata.eve.tls.ja3.string": "718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3", + "suricata.eve.tls.ja3s.hash": "391231ba5675e42807b9e1f457b2614e", + "suricata.eve.tls.ja3s.string": "742,48172,30210-30", + "suricata.eve.tls.notafter": "2026-06-25T17:36:29", + "suricata.eve.tls.notbefore": "2016-06-27T17:36:29", + "suricata.eve.tls.serial": "72:A9:2C:51", + "suricata.eve.tls.sni": "host.domain.net", + "suricata.eve.tls.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "suricata.eve.tls.version": "TLS 1.2", + "tags": [ + "suricata" + ], + "tls.client.ja3": "3f1ea03f5822e8021b60cc3e4b233181", + "tls.client.server_name": "host.domain.net", + "tls.server.hash.sha1": "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC", + "tls.server.issuer": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "tls.server.ja3s": "391231ba5675e42807b9e1f457b2614e", + "tls.server.not_after": "2026-06-25T17:36:29", + "tls.server.not_before": "2016-06-27T17:36:29", + "tls.server.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "tls.version": "1.2", + "tls.version_protocol": "tls" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index ec02bba8dd1..5d44c5bd12f 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -43,6 +43,7 @@ "@timestamp": "2018-07-05T19:07:20.910Z", "destination.address": "192.168.156.70", "destination.bytes": 343, + "destination.domain": "l2.io", "destination.ip": "192.168.156.70", "destination.packets": 3, "destination.port": 443, @@ -96,6 +97,8 @@ "tags": [ "suricata" ], + "tls.client.server_name": "l2.io", + "tls.resumed": true, "tls.version": "1.2", "tls.version_protocol": "tls" }, diff --git a/x-pack/filebeat/module/suricata/fields.go b/x-pack/filebeat/module/suricata/fields.go index 578c30c9d6b..4cba3d5ee74 100644 --- a/x-pack/filebeat/module/suricata/fields.go +++ b/x-pack/filebeat/module/suricata/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSuricata returns asset data. // This is the base64 encoded gzipped contents of module/suricata. func AssetSuricata() string { - return "eJzsXEuP27oV3udXaDerGE3QBMUsim7SRYG2iwDdEsfkkcyar5CUPe6vLyR7PLJFyuIDc3Fzk1UwCT+e94tH87HZ4+m5cb3lFDx8aBrPvcDn5vvbTxg6arnxXKvn5q8fmqZp/qlZL7BptW12oJjgqmv8Dptv//nW/OP7v//VCN25xljNeoqs2Z6ueJsPTdNyFMw9j0gfGwUSbygY/viTweems7o3l58EqBj+/H3Ealqr5UjB6z0jKUJ3TcsFbi7/fXrx9HI84PVnobsX7p/QgC9GW39mdyaMyYF7Ku4oUZ4MFNz88ytRezwdtWXXfwtigDHEWO010ZZ3+TiemuDhe8nEeLpBIq2AGymsJucNxrhdBGCrtUBQjwCudBBPy0gBui8jxZ1UIYAHf28kiUxMxFGoGet8GTctzxLH23GBXLW6kr26HXwqE8hA0PC3CAoIDnORG/C789HN8NeH6nshnEUuEFp172BBzmuLMRpWqr4DE7O+ta6wg89fvpZxItmXQlHw/2Vr++Zs0MA5lSaeFuLafj3P0HkypKfg+RCBZ/KGc1zBkPU2N8eDtzhLcy5xurcUV+CPWS0RXKE/arvfeAvKrbiCgiFUhVl4LOdBAjycNh/yPzkX1qG6P56dj+eGlBrwbQ2MhQi5FoO91YyZEKVh1HtRcNpSzQplkEH9NU0JfZxzn1AjogR+z3521vXg+6xC5CbKJbvfNMY98sExDgdUtlbgO+9rFdWL4lpKOQMNG4vOaOVwc4a55Slmq8i4RRor9Vaa63A96R1aAh3OAu0aBt4Ob4YOhysQjy4dMwfVMU9NId1iizZf7D96dH4zgtgJTuTG3sZIXhSQFasls9POZ1aqwzVMS+APKzSBqvOx5m29rW41O222J49ulaYk+p2OBcfVqrpDWbqQauUj/Xta580lOg8yNZI9/e168ulBHFOEt0BLpgwCI4Veeiij4LHT9lRYgeMBLfcxlCWFj2OXzQwgGgcPBRm/K2vaeKfA97awZgA6pLtsSeneUy0fJowrsTkV1hVkNvXJNjPB4wknBBcDvMsr5IDWhcS5UikTXnXrj2CxFPHNJewhmqx+Io6nNVGtJomCWfCzPOHt0SoUxADdY2DmtiYQzMCY1YEBShYUb/PBrhWLGXJQQYRj6OM1Zp7YQ7kqjahA3V5GkkRJYX4sTW0SZe/msk5hrOUCyTjGq8qeNqjIgF1mTAxbC/dPGIWChxcygJIdL/ZAbg5/jkKEyYsTeFv+6T5I3Tr6Juq10ElUVbAsgnMotyIw8V2LNpHcfGD7S3KP3VXoY1V38HToaHOjyCTsswrBrDwiokTbEakZElQeQ48T2YA6VEqloc2fVFMROJVLrpMAEo9c60CcgUDSePcEFhJpUX3rsGeatMDD3pokIqfI2azHKq1Ua8qh9YSBh9EWBZiRzFIi0Q2Fd2kuPMutlJaTCr2s58tJaStBVBFT4MH+3Yx9ytwBBGeE7pDuXS+LlT8muYpWerHMOjoU3PkaygvtJyQKylsESRgavyMWge6Ko8O1KjiRKqYxweuKFfAaYBhvWxJ8ZEvDU5oEa5e0boBqVnmcAYeOmL0nwef7NBZvB9N5GOUFgoU8IU8iuSml4SBAkR9c/SjEeVK9EE+FIMIPKFGQ/GajV64355W/8MR/LaUT0e898VoTJyFIclauKJRfV1zlLbbJFfrFlWa9r9HvfyVckQrUcLPkqPlG+VofcLMwUU2OB/q3bw/Z7LGqqSCuug43xL3SOoMGBo2JdSa81EppYwddWtRLI0odzy0E8N9b34zWmWIzQb9Dq7C0qh0ic62Ahoh/+dPnTzB/5k+q8QKNaPmcnXRCb6HUhi5Y4b3Z95/cC30ksqtbDlt9dGTbu/kjehp/A3GOXMaxVbCUroM2cuj23JjiBo4K7ZARY3tVjKXwWAfoLC2LUh+KsbYnM7SUlVgcV5DrsXgxB8JVeet8RhxnKuVt/QAl4UVgaZC3lYlCaQLbMSkxB4whAk6Ve/DFwUDBY1Cw7Eh+BwrWY6kowffjDGLGqTQJV9OpYEy5ikjhyUUqkpM15VSHJunye6ilNaYcFC6Dk71khVG0htbS/hmslgFss1Gujv/yK5r8wQNAHW/7+dwkbs1x/1hek1Pab7GN7w+xabMUweDO9WhZbB917bKj4mUAsYFZGhXn51Ri0fWy9NvIlqsOrbE8ura6fhGUB/rfJAylPbQ+WvutUbTrt//N/IBkWoKed1Ajm56rUAIlZ64DjM8+xOvF7eKllerp90erPm5wHgLrjI8vumy535yOXGH2vhpP99uukSuhK9karfAx86siF5eml5i+fMe5SocWwZWGmouSSuldqR9UsWC2xvPHFdy8cDjz/KDjPv4CePYNWDgzBj4HXZtVJ7Hp/vdLJMSmQAmUG5ssNZ7MBJZoZhK4IK3V812TJJgdiixC5sLFF4N0bkxJ4X/+y0gey/j/AQAA//93dIZ1" + return "eJzsXEuP67YV3t9fod2sYjRpExSzKLpJFwXaLgJ0SxyTRxJjvi5J2eP++kKyxyNbpCw+MEWTO6uBZ/jxvHhePPR3zQHPr40bLKfg4UvTeO4Fvja/fHzC0FHLjedavTZ/+dI0TfMPzQaBTatt04Nigquu8T02P//75+bvv/zrn43QnWuM1WygyJr9+Ya3+9I0LUfB3OuE9F2jQOIdBeOPPxt8bTqrB3P9JEDF+PO3CatprZYTBe/7TKQI3TUtF7i7/vt84/nmeMTbZ6G9V/af0YBvRlt/YXchjNmCRyoeKFGejBTc/fmdqAOeT9qy29+CGGAMMVZ7TbTlXT6Opya4+FEyMZ7ukEgr4E4Km8n5gDGujwDstRYI6hnAjQ7iaRkpQA9lpLizKgTw4B+NJJGJmTgKNWOdL+Om5Vni+FgukKtWV7JX18P3ZQIZCRp/i6CA4LAUuQHfX5buxl+fqu+NcBbZQGjVfYIFOa8txmjYqPoOTMz6th6FHn748acyTiT7sVAU/D/Z2r5bGzRwTqWJh4W4tt/XM3SejOEpuD5E4IW8cR1XMEa93d3y4C7O0pxNnB4sxQ34U1RLBFfoT9oedt6Cchu2oGAIVWEWnst5lAAPh82n/M/WhXWoHpdnx+OlIaU6fFsDY8VDbsVgHzljJkSpG/VeFKy2VLNCGWRQfwtTQp+W3CfkiCiBP7KfHXU9+CErEbnzcsnHb+7jnp3ByQ8HVLZV4L33tZLqVXGthZyRhp1FZ7RyuLvA3PMUs1Vk3CKNpXobzXXcngwOLYEOF452CwMfi3djhcMViGebTpGD6thJTSHdYos2X+xfB3R+N4HYGU5kx8HGSF4VkBWbJdNr5zMz1XEbpiXwpxmaQNX5WPG23Vb3mp13+7NHt0lTEn2vY85xs6oeUNY2pFr5SP2eVnlzic6DTPVkL3+9rXx54scU4S3Qki6DwEiil+7KKHjstD0XZuB4RMt9DGVN4VPbZbcAiPrBY0HE78qKNt4p8IMtzBmAjuEuW1J68FTLpwHjRmxOhnUDWXR9ss1M8HjACcHFAB/iCjmidSFxblTKjFfd+hNYLEX8OBL2GA1WvyGO5zlRrSKJglk5Z3nCO6BVKIgBesBAz22LI1iAMasDDZQsKN7mg90yFjPGoAIPx9DHc8w8sYdiVRpRgby9jCSJksJyWZraJMrBLWWdwljLBZKpjVeVPW1QkRG7zJgYthYerzAKBQ9vZAQlPS8+gdwc/xSFCJMXJ/A+/dNDkLpt9M3Ua6GTqKpgWQTnUO5FoOO7FW0muWXD9pvknh9XoU9Vj4OnY0Wb60Vmbp9VcGblHhEl2o5IzZCg8hi6nMgG1KFUKg1teaWaisCpXDs6CSBxz7UNxBkIBI1PD2AhkRbltw4HpkkLPHxak0TkFLmY9ZSllWpNObSeMPAw2aIAM5FZSiS6MfEujYUXuZXSclahm/V8OSltJYgqYgpc2H+asc+ZO4LgjNAe6cENslj5U5CraKVXy6yjQ8Gdr6G80HxCoqC8RZCEofE9sQi0L/YOt6zgTKqYxgyvK1bAu4NhvG1J8JItDU9pEsxd0qoBqlnldgYcO2IOngSv79NYvG9M52GUJwgW8oQ88+SmlIajAEW+cvW1EOdFDUK8FIIIP6JEQfKLjUG5wVxG/sId/62UzkR/8MRrTZyEIMlZsaJQfl1xlrdaJleoFzea9aFGvf8T4YpUoIabtYOab5Tv+QE3Kx3VZH+g//flIVtcVjUVxFX3wI1+rzTPoIFGY2KeCW+1QtpUQZcm9dKI0oPnVhz4/1vdjNaZYjNB36NVWJrVjp65lkNDxD//4YfvYXnNn5TjBQrR8j476YTeQ6kNXbHCc7Of37kX+kRkVzcdtvrkyH5wy0v0NP5G4hy5tmOrYCldB23i0B24McUFHBXaISPGDqoYS+GpDtBFWhalPhZj7c9mLCkrsTiNINdj8WoOhKvy0vmCOPVUysv6EUrCm8BSJ28rE4XSBKZjUnwOGEMEnCvX4KuNgYLLoGDakXwPFMzHUlGC98cZxExdaRLOplPBmHIVkcKdi1QkJ2vKqQ5N0uXXUGtjTDkoXAY7e8kKo2gNraX9C1gtA9hno9wO/ts3b/I7dwB1Tttv75jErTl+PtbH5JT2e2zj80NsXixFMLhzA1oWm0fdOuyoeBlArGGWRsXlOpVYdIMsfRvZctWhNZZHx1a3D4LyQP2bhKG0h9ZHc78tinbD/tfiByS/wh/TegQMWxiEJ5Mhj8mBCBQMzyOA85ar5TBanP4lRA+rfmm7AH6H/M9rkMsQcmTUdxNKoObI9YDTvR/xenW8fG2mfv4AbdPrFuchMM/6fKPrM4e71ZEtzMFX4+lx3DmyJXQlY8MVXrO/K3J1an6N6etD3k06tAiuNNZclVRK70b9oIpFsy2uf5rBzouHi5MfPLjPn4AvHgGGU6PAe+CtadXMNz1+wUiCbwrkwLm+yVLjyUJgiWYmgQvSWr0cNkqC6VFkEbIULr4ZpEtjSnL/y2+jeS7j/wYAAP//9F32EA==" } diff --git a/x-pack/libbeat/common/cloudfoundry/cache.go b/x-pack/libbeat/common/cloudfoundry/cache.go index bc87fb11836..22f41f3b23c 100644 --- a/x-pack/libbeat/common/cloudfoundry/cache.go +++ b/x-pack/libbeat/common/cloudfoundry/cache.go @@ -22,43 +22,57 @@ type cfClient interface { // clientCacheWrap wraps the cloudfoundry client to add a cache in front of GetAppByGuid. type clientCacheWrap struct { - cache *common.Cache - client cfClient - log *logp.Logger + cache *common.Cache + client cfClient + log *logp.Logger + errorTTL time.Duration } // newClientCacheWrap creates a new cache for application data. -func newClientCacheWrap(client cfClient, ttl time.Duration, log *logp.Logger) *clientCacheWrap { +func newClientCacheWrap(client cfClient, ttl time.Duration, errorTTL time.Duration, log *logp.Logger) *clientCacheWrap { return &clientCacheWrap{ - cache: common.NewCacheWithExpireOnAdd(ttl, 100), - client: client, - log: log, + cache: common.NewCacheWithExpireOnAdd(ttl, 100), + client: client, + errorTTL: errorTTL, + log: log, } } +type appResponse struct { + app *cfclient.App + err error +} + // fetchApp uses the cfClient to retrieve an App entity and // stores it in the internal cache func (c *clientCacheWrap) fetchAppByGuid(guid string) (*cfclient.App, error) { app, err := c.client.GetAppByGuid(guid) + resp := appResponse{ + app: &app, + err: err, + } + timeout := time.Duration(0) if err != nil { - return nil, err + // Cache nil, because is what we want to return when there was an error + resp.app = nil + timeout = c.errorTTL } - c.cache.Put(app.Guid, &app) - return &app, nil + c.cache.PutWithTimeout(guid, &resp, timeout) + return resp.app, resp.err } // GetApp returns CF Application info, either from the cache or // using the CF client. func (c *clientCacheWrap) GetAppByGuid(guid string) (*cfclient.App, error) { - cachedApp := c.cache.Get(guid) - if cachedApp == nil { + cachedResp := c.cache.Get(guid) + if cachedResp == nil { return c.fetchAppByGuid(guid) } - app, ok := cachedApp.(*cfclient.App) + resp, ok := cachedResp.(*appResponse) if !ok { - return nil, fmt.Errorf("error converting cached app") + return nil, fmt.Errorf("error converting cached app response (of type %T), this is likely a bug", cachedResp) } - return app, nil + return resp.app, resp.err } // StartJanitor starts a goroutine that will periodically clean the applications cache. diff --git a/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go b/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go new file mode 100644 index 00000000000..f6af11787c9 --- /dev/null +++ b/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build cloudfoundry + +package cloudfoundry + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudfoundry-community/go-cfclient" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" +) + +func TestGetApps(t *testing.T) { + var conf Config + err := common.MustNewConfigFrom(cftest.GetConfigFromEnv(t)).Unpack(&conf) + require.NoError(t, err) + + log := logp.NewLogger("cloudfoundry") + hub := NewHub(&conf, "filebeat", log) + + client, err := hub.Client() + require.NoError(t, err) + apps, err := client.(*clientCacheWrap).client.(*cfclient.Client).ListApps() + require.NoError(t, err) + + t.Logf("%d applications available", len(apps)) + + t.Run("request one of the available applications", func(t *testing.T) { + if len(apps) == 0 { + t.Skip("no apps in account?") + } + client, err := hub.Client() + require.NoError(t, err) + + guid := apps[0].Guid + app, err := client.GetAppByGuid(guid) + assert.Equal(t, guid, app.Guid) + assert.NoError(t, err) + }) + + t.Run("handle error when application is not available", func(t *testing.T) { + client, err := hub.Client() + require.NoError(t, err) + + testNotExists := func(t *testing.T) { + app, err := client.GetAppByGuid("notexists") + assert.Nil(t, app) + assert.True(t, cfclient.IsAppNotFoundError(err)) + } + + var firstTimeDuration time.Duration + t.Run("first call", func(t *testing.T) { + startTime := time.Now() + testNotExists(t) + firstTimeDuration = time.Now().Sub(startTime) + }) + + t.Run("second call, in cache, faster, same response", func(t *testing.T) { + for i := 0; i < 10; i++ { + startTime := time.Now() + testNotExists(t) + require.True(t, firstTimeDuration > time.Now().Sub(startTime)) + } + }) + }) +} diff --git a/x-pack/libbeat/common/cloudfoundry/cache_test.go b/x-pack/libbeat/common/cloudfoundry/cache_test.go index 678dd74ecfe..9e18a5ac86e 100644 --- a/x-pack/libbeat/common/cloudfoundry/cache_test.go +++ b/x-pack/libbeat/common/cloudfoundry/cache_test.go @@ -7,7 +7,6 @@ package cloudfoundry import ( - "fmt" "testing" "time" @@ -26,17 +25,27 @@ func TestClientCacheWrap(t *testing.T) { Memory: 1, // use this field to track if from cache or from client } fakeClient := &fakeCFClient{app, 0} - cache := newClientCacheWrap(fakeClient, ttl, logp.NewLogger("cloudfoundry")) + cache := newClientCacheWrap(fakeClient, ttl, ttl, logp.NewLogger("cloudfoundry")) + + missingAppGuid := mustCreateFakeGuid() // should err; different app client doesn't have - _, err := cache.GetAppByGuid(mustCreateFakeGuid()) - assert.Error(t, err) + one, err := cache.GetAppByGuid(missingAppGuid) + assert.Nil(t, one) + assert.True(t, cfclient.IsAppNotFoundError(err)) + assert.Equal(t, 1, fakeClient.callCount) + + // calling again; the miss should be cached + one, err = cache.GetAppByGuid(missingAppGuid) + assert.Nil(t, one) + assert.True(t, cfclient.IsAppNotFoundError(err)) + assert.Equal(t, 1, fakeClient.callCount) // fetched from client for the first time - one, err := cache.GetAppByGuid(guid) + one, err = cache.GetAppByGuid(guid) assert.NoError(t, err) assert.Equal(t, app, *one) - assert.Equal(t, 1, fakeClient.callCount) + assert.Equal(t, 2, fakeClient.callCount) // updated app in fake client, new fetch should not have updated app updatedApp := cfclient.App{ @@ -47,14 +56,14 @@ func TestClientCacheWrap(t *testing.T) { two, err := cache.GetAppByGuid(guid) assert.NoError(t, err) assert.Equal(t, app, *two) - assert.Equal(t, 1, fakeClient.callCount) + assert.Equal(t, 2, fakeClient.callCount) // wait the ttl, then it should have updated app time.Sleep(ttl) three, err := cache.GetAppByGuid(guid) assert.NoError(t, err) assert.Equal(t, updatedApp, *three) - assert.Equal(t, 2, fakeClient.callCount) + assert.Equal(t, 3, fakeClient.callCount) } type fakeCFClient struct { @@ -63,10 +72,10 @@ type fakeCFClient struct { } func (f *fakeCFClient) GetAppByGuid(guid string) (cfclient.App, error) { + f.callCount++ if f.app.Guid != guid { - return f.app, fmt.Errorf("no app with guid") + return cfclient.App{}, notFoundError() } - f.callCount++ return f.app, nil } @@ -77,3 +86,8 @@ func mustCreateFakeGuid() string { } return uuid.String() } + +// notFoundError returns a cloud foundry error that satisfies cfclient.IsAppNotFoundError(err) +func notFoundError() error { + return cfclient.CloudFoundryError{Code: 100004} +} diff --git a/x-pack/libbeat/common/cloudfoundry/config.go b/x-pack/libbeat/common/cloudfoundry/config.go index 8f1139bd7a2..0724bdc66e1 100644 --- a/x-pack/libbeat/common/cloudfoundry/config.go +++ b/x-pack/libbeat/common/cloudfoundry/config.go @@ -41,8 +41,11 @@ type Config struct { // multiple filebeats will shard the load of receiving and sending events. ShardID string `config:"shard_id"` - // Maximum amount of time to cache application objects from CF client + // Maximum amount of time to cache application objects from CF client. CacheDuration time.Duration `config:"cache_duration"` + + // Time to wait before retrying to get application info in case of error. + CacheRetryDelay time.Duration `config:"cache_retry_delay"` } // InitDefaults initialize the defaults for the configuration. @@ -55,6 +58,7 @@ func (c *Config) InitDefaults() { } c.ShardID = uuid.String() c.CacheDuration = 120 * time.Second + c.CacheRetryDelay = 20 * time.Second c.Version = ConsumerVersionV1 } diff --git a/x-pack/libbeat/common/cloudfoundry/hub.go b/x-pack/libbeat/common/cloudfoundry/hub.go index 823087ea959..4bb7fce1eec 100644 --- a/x-pack/libbeat/common/cloudfoundry/hub.go +++ b/x-pack/libbeat/common/cloudfoundry/hub.go @@ -67,7 +67,7 @@ func (h *Hub) Client() (Client, error) { if h.cfg.UaaAddress != "" { cf.Endpoint.AuthEndpoint = h.cfg.UaaAddress } - return newClientCacheWrap(cf, h.cfg.CacheDuration, h.log), nil + return newClientCacheWrap(cf, h.cfg.CacheDuration, h.cfg.CacheRetryDelay, h.log), nil } // RlpListener returns a listener client that calls the passed callback when the provided events are streamed through diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go index a80cc24700c..d18a04ca979 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go @@ -60,7 +60,7 @@ func (d *addCloudFoundryMetadata) Run(event *beat.Event) (*beat.Event, error) { } valI, err := event.GetValue("cloudfoundry.app.id") if err != nil { - // doesn't have the required cf.app.id value to add more information + // doesn't have the required cloudfoundry.app.id value to add more information return event, nil } val, _ := valI.(string) diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go index ca83a3e24c2..1aff4cb2df8 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go @@ -5,7 +5,6 @@ package add_cloudfoundry_metadata import ( - "fmt" "testing" "time" @@ -148,7 +147,7 @@ type fakeClient struct { func (c *fakeClient) GetAppByGuid(guid string) (*cfclient.App, error) { if c.app.Guid != guid { - return nil, fmt.Errorf("unknown app") + return nil, cfclient.CloudFoundryError{Code: 100004} } return &c.app, nil } diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc index 7c5b2daba96..558b5a1031b 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc @@ -53,6 +53,8 @@ It has the following settings: `client_secret`:: Client Secret to authenticate with Cloud Foundry. -`cache_duration`:: (Optional) Maximum amount of time to cache an application's metadata. +`cache_duration`:: (Optional) Maximum amount of time to cache an application's metadata. Defaults to 120 seconds. + +`cache_retry_delay`:: (Optional) Time to wait before trying to obtain an application's metadata again in case of error. Defaults to 20 seconds. `ssl`:: (Optional) SSL configuration to use when connecting to Cloud Foundry. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 2cc447723be..aac756190f2 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -615,6 +615,13 @@ func insertTags(events map[string]mb.Event, identifier string, resourceTagMap ma subIdentifiers := strings.Split(identifier, dimensionSeparator) for _, v := range subIdentifiers { tags := resourceTagMap[v] + // some metric dimension values are arn format, eg: AWS/DDOS namespace metric + if len(tags) == 0 && strings.HasPrefix(v, "arn:") { + resourceID, err := aws.FindIdentifierFromARN(v) + if err == nil { + tags = resourceTagMap[resourceID] + } + } if len(tags) != 0 { // By default, replace dot "." using underscore "_" for tag keys. // Note: tag values are not dedotted. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index e6b5cdebe5a..93a0c42492c 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1407,10 +1407,14 @@ func TestInsertTags(t *testing.T) { tagValue1 := "engineering" tagKey2 := "owner" tagValue2 := "foo" + identifierContainsArn := "arn:aws:ec2:ap-northeast-1:111111111111:eip-allocation/eipalloc-0123456789abcdef,SYNFlood" + tagKey3 := "env" + tagValue3 := "dev" events := map[string]mb.Event{} events[identifier1] = aws.InitEvent(regionName, accountName, accountID) events[identifier2] = aws.InitEvent(regionName, accountName, accountID) + events[identifierContainsArn] = aws.InitEvent(regionName, accountName, accountID) resourceTagMap := map[string][]resourcegroupstaggingapi.Tag{} resourceTagMap["test-s3-1"] = []resourcegroupstaggingapi.Tag{ @@ -1425,6 +1429,12 @@ func TestInsertTags(t *testing.T) { Value: awssdk.String(tagValue2), }, } + resourceTagMap["eipalloc-0123456789abcdef"] = []resourcegroupstaggingapi.Tag{ + { + Key: awssdk.String(tagKey3), + Value: awssdk.String(tagValue3), + }, + } cases := []struct { title string @@ -1444,6 +1454,12 @@ func TestInsertTags(t *testing.T) { "aws.tags.owner", tagValue2, }, + { + "test identifier with arn value", + identifierContainsArn, + "aws.tags.env", + tagValue3, + }, } for _, c := range cases { diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index f9e0f58b16c..4c92cb140c6 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -188,9 +188,9 @@ func GetResourcesTags(svc resourcegroupstaggingapiiface.ClientAPI, resourceTypeF } for _, resourceTag := range output.ResourceTagMappingList { - identifier, err := findIdentifierFromARN(*resourceTag.ResourceARN) + identifier, err := FindIdentifierFromARN(*resourceTag.ResourceARN) if err != nil { - err = errors.Wrap(err, "error findIdentifierFromARN") + err = errors.Wrap(err, "error FindIdentifierFromARN") return nil, err } resourceTagMap[identifier] = resourceTag.Tags @@ -199,7 +199,7 @@ func GetResourcesTags(svc resourcegroupstaggingapiiface.ClientAPI, resourceTypeF return resourceTagMap, nil } -func findIdentifierFromARN(resourceARN string) (string, error) { +func FindIdentifierFromARN(resourceARN string) (string, error) { arnParsed, err := arn.Parse(resourceARN) if err != nil { err = errors.Wrap(err, "error Parse arn") diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index c33d61c641f..4270d52b6ef 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -377,7 +377,7 @@ func TestFindIdentifierFromARN(t *testing.T) { } for _, c := range cases { - identifier, err := findIdentifierFromARN(c.resourceARN) + identifier, err := FindIdentifierFromARN(c.resourceARN) assert.NoError(t, err) assert.Equal(t, c.expectedIdentifier, identifier) } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc index 47ff699566c..2f42a919e3d 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc @@ -16,19 +16,20 @@ call. [float] == Metricset config and parameters -* *metric_types*: Required, a list of metric type strings. Each call of the +* *metric_types*: Required, a list of metric type strings, or a list of metric +type prefixes. For example, `instance/cpu` is the prefix for metric type +`instance/cpu/usage_time`, `instance/cpu/utilization` etc Each call of the `ListTimeSeries` API can return any number of time series from a single metric type. Metric type is to used for identifying a specific time series. * *aligner*: A single string with which aggregation operation need to be applied onto time series data for ListTimeSeries API. If it's not given, default aligner -is set to be `ALIGN_NONE`. Google Cloud also supports `ALIGN_DELTA`, `ALIGN_RATE`, -`ALIGN_MIN`, `ALIGN_MAX`, `ALIGN_MEAN`, `ALIGN_COUNT`, `ALIGN_SUM` and etc. +is `ALIGN_NONE`. Google Cloud also supports `ALIGN_DELTA`, `ALIGN_RATE`, +`ALIGN_MIN`, `ALIGN_MAX`, `ALIGN_MEAN`, `ALIGN_COUNT`, `ALIGN_SUM` etc. Please see https://cloud.google.com/monitoring/api/ref_v3/rpc/google.monitoring.v3#aligner[Aggregation Aligner] for the full list of aligners. - [float] === Example Configuration * `stackdriver` metricset is enabled to collect metrics from all zones under @@ -37,7 +38,7 @@ are specified: first one is to collect CPU usage time and utilization with aggregation aligner ALIGN_MEAN; second one is to collect uptime with aggregation aligner ALIGN_SUM. These metric types all have 240 seconds ingest delay time and 60 seconds sample period. With `period` specified as `300s` in the config below, -Metricbeat will collect compute metrics from googlecloud every 5-minute with +Metricbeat will collect compute metrics from Google Cloud every 5-minute with given aggregation aligner applied for each metric type. + [source,yaml] @@ -69,7 +70,7 @@ are specified: first one is to collect CPU usage time and utilization with aggregation aligner ALIGN_MEAN; second one is to collect uptime with aggregation aligner ALIGN_SUM. These metric types all have 240 seconds ingest delay time and 60 seconds sample period. With `period` specified as `60s` in the config below, -Metricbeat will collect compute metrics from googlecloud every minute with no +Metricbeat will collect compute metrics from Google Cloud every minute with no aggregation. This case, the aligners specified in the configuration will be ignored. + @@ -94,3 +95,32 @@ ignored. metric_types: - "instance/uptime" ---- + +* `stackdriver` metricset is enabled to collect metrics from all zones under +`europe-west1-c` region in `elastic-observability` project. One set of metrics +will be collected: metric types that starts with `instance/cpu` under `compute` +service with aligner ALIGN_NONE. These metric types all have 240 seconds ingest +delay time and 60 seconds sample period. With `period` specified as `1m` in +the config below, Metricbeat will collect compute metrics from Google Cloud +every minute with no aggregation. The metric types in `compute` service with +`instance/cpu` prefix are: `instance/cpu/reserved_cores`, +`instance/cpu/scheduler_wait_time`, `instance/cpu/usage_time`, and +`instance/cpu/utilization`. + ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - stackdriver + zone: "europe-west1-c" + project_id: elastic-observability + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + metrics: + - aligner: ALIGN_NONE + service: compute + metric_types: + - "instance/cpu" +---- diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 4b48cf06ff0..a3f9a325038 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -76,17 +76,16 @@ func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, sdc stackDriv results := make([]timeSeriesWithAligner, 0) aligner := sdc.Aligner - serviceName := sdc.ServiceName - for _, mt := range sdc.MetricTypes { + for mt, meta := range metricsMeta { wg.Add(1) + metricMeta := meta go func(mt string) { defer wg.Done() - metricMeta := metricsMeta[mt] r.logger.Debugf("For metricType %s, metricMeta = %s", mt, metricMeta) interval, aligner := getTimeIntervalAligner(metricMeta.ingestDelay, metricMeta.samplePeriod, r.config.period, aligner) - ts := r.Metric(ctx, serviceName+".googleapis.com/"+mt, interval, aligner) + ts := r.Metric(ctx, mt, interval, aligner) lock.Lock() defer lock.Unlock() results = append(results, ts) diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go index 053db176f21..a81bca2b86e 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go @@ -9,13 +9,12 @@ import ( "fmt" "time" - "github.com/golang/protobuf/ptypes/duration" - monitoring "cloud.google.com/go/monitoring/apiv3" - + "github.com/golang/protobuf/ptypes/duration" "github.com/pkg/errors" - + "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/genproto/googleapis/api/metric" monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" "github.com/elastic/beats/v7/libbeat/common" @@ -216,41 +215,54 @@ func (mc *stackDriverConfig) Validate() error { // (sample period and ingest delay) of each given metric type func (m *MetricSet) metricDescriptor(ctx context.Context, client *monitoring.MetricClient) (map[string]metricMeta, error) { metricsWithMeta := make(map[string]metricMeta, 0) + req := &monitoringpb.ListMetricDescriptorsRequest{ + Name: "projects/" + m.config.ProjectID, + } for _, sdc := range m.stackDriverConfig { for _, mt := range sdc.MetricTypes { - req := &monitoringpb.ListMetricDescriptorsRequest{ - Name: "projects/" + m.config.ProjectID, - Filter: fmt.Sprintf(`metric.type = "%s"`, sdc.ServiceName+".googleapis.com/"+mt), - } - + req.Filter = fmt.Sprintf(`metric.type = starts_with("%s")`, sdc.ServiceName+".googleapis.com/"+mt) it := client.ListMetricDescriptors(ctx, req) - out, err := it.Next() - if err != nil { - err = errors.Errorf("Could not make ListMetricDescriptors request: %s: %v", mt, err) - m.Logger().Error(err) - return metricsWithMeta, err - } - // Set samplePeriod default to 60 seconds and ingestDelay default to 0. - meta := metricMeta{ - samplePeriod: 60 * time.Second, - ingestDelay: 0 * time.Second, + for { + out, err := it.Next() + if err != nil && err != iterator.Done { + err = errors.Errorf("Could not make ListMetricDescriptors request for metric type %s: %v", mt, err) + m.Logger().Error(err) + return metricsWithMeta, err + } + + if out != nil { + metricsWithMeta = m.getMetadata(out, metricsWithMeta) + } + + if err == iterator.Done { + break + } } + } + } - if out.Metadata.SamplePeriod != nil { - m.Logger().Debugf("For metric type %s: sample period = %s", mt, out.Metadata.SamplePeriod) - meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second - } + return metricsWithMeta, nil +} - if out.Metadata.IngestDelay != nil { - m.Logger().Debugf("For metric type %s: ingest delay = %s", mt, out.Metadata.IngestDelay) - meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second - } +func (m *MetricSet) getMetadata(out *metric.MetricDescriptor, metricsWithMeta map[string]metricMeta) map[string]metricMeta { + // Set samplePeriod default to 60 seconds and ingestDelay default to 0. + meta := metricMeta{ + samplePeriod: 60 * time.Second, + ingestDelay: 0 * time.Second, + } - metricsWithMeta[mt] = meta - } + if out.Metadata.SamplePeriod != nil { + m.Logger().Debugf("For metric type %s: sample period = %s", out.Type, out.Metadata.SamplePeriod) + meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second } - return metricsWithMeta, nil + if out.Metadata.IngestDelay != nil { + m.Logger().Debugf("For metric type %s: ingest delay = %s", out.Type, out.Metadata.IngestDelay) + meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second + } + + metricsWithMeta[out.Type] = meta + return metricsWithMeta }