aws payload tagging (#4309)

Co-authored-by: Jordi Bertran de Balanda <jordi.bertran@datadoghq.com>

LICENSE-3rdparty.csv

@@ -14,6 +14,7 @@ require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
 require,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki
 require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc.
 require,jest-docblock,MIT,Copyright Meta Platforms, Inc. and affiliates.
+require,jsonpath-plus,MIT,Copyright (c) 2011-2019 Stefan Goessner, Subbu Allamaraju, Mike Brevoort, Robert Krahn, Brett Zamir, Richard Schneider
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward
 require,limiter,MIT,Copyright 2011 John Hurliman
 require,lodash.sortby,MIT,Copyright JS Foundation and other contributors
@@ -26,6 +27,7 @@ require,pprof-format,MIT,Copyright 2022 Stephen Belanger
 require,protobufjs,BSD-3-Clause,Copyright 2016 Daniel Wirtz
 require,tlhunter-sorted-set,MIT,Copyright (c) 2023 Datadog Inc.
 require,retry,MIT,Copyright 2011 Tim Koschützki Felix Geisendörfer
+require,rfdc,MIT,Copyright 2019 David Mark Clements
 require,semver,ISC,Copyright Isaac Z. Schlueter and Contributors
 require,shell-quote,mit,Copyright (c) 2013 James Halliday
 dev,@types/node,MIT,Copyright Authors

index.d.ts

@@ -729,6 +729,26 @@ declare namespace tracer {
      * The selection and priority order of context propagation injection and extraction mechanisms.
      */
     propagationStyle?: string[] | PropagationStyle
+
+    /**
+     * Cloud payload report as tags
+     */
+    cloudPayloadTagging?: {
+      /**
+       *  Additional JSONPath queries to replace with `redacted` in request payloads
+       *  Undefined or invalid JSONPath queries disable the feature for requests.
+       */
+      request?: string,
+      /**
+       *  Additional JSONPath queries to replace with `redacted` in response payloads
+       *  Undefined or invalid JSONPath queries disable the feature for responses.
+       */
+      response?: string,
+      /**
+       *  Maximum depth of payload traversal for tags
+       */
+      maxDepth?: number
+    }
   }
 
   /**

package.json

@@ -13,6 +13,7 @@
     "type:doc": "cd docs && yarn && yarn build",
     "type:test": "cd docs && yarn && yarn test",
     "lint": "node scripts/check_licenses.js && eslint . && yarn audit --groups dependencies",
+    "lint-fix": "node scripts/check_licenses.js && eslint . --fix && yarn audit --groups dependencies",
     "services": "node ./scripts/install_plugin_modules && node packages/dd-trace/test/setup/services",
     "test": "SERVICES=* yarn services && mocha --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'",
     "test:appsec": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" \"packages/dd-trace/test/appsec/**/*.spec.js\"",
@@ -87,6 +88,7 @@
     "int64-buffer": "^0.1.9",
     "istanbul-lib-coverage": "3.2.0",
     "jest-docblock": "^29.7.0",
+    "jsonpath-plus": "^9.0.0",
     "koalas": "^1.0.2",
     "limiter": "1.1.5",
     "lodash.sortby": "^4.7.0",
@@ -98,6 +100,7 @@
     "pprof-format": "^2.1.0",
     "protobufjs": "^7.2.5",
     "retry": "^0.13.1",
+    "rfdc": "^1.3.1",
     "semver": "^7.5.4",
     "shell-quote": "^1.8.1",
     "tlhunter-sorted-set": "^0.1.0"
@@ -116,11 +119,11 @@
     "dotenv": "16.3.1",
     "esbuild": "0.16.12",
     "eslint": "^8.57.0",
-    "eslint-config-standard": "^17.1.0",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-mocha": "^10.4.3",
     "eslint-plugin-n": "^16.6.2",
     "eslint-plugin-promise": "^6.4.0",
+    "eslint-config-standard": "^17.1.0",
     "express": "^4.18.2",
     "get-port": "^3.2.0",
     "glob": "^7.1.6",

packages/datadog-plugin-aws-sdk/src/base.js

@@ -5,9 +5,11 @@ const ClientPlugin = require('../../dd-trace/src/plugins/client')
 const { storage } = require('../../datadog-core')
 const { isTrue } = require('../../dd-trace/src/util')
 const coalesce = require('koalas')
+const { tagsFromRequest, tagsFromResponse } = require('../../dd-trace/src/payload-tagging')
 
 class BaseAwsSdkPlugin extends ClientPlugin {
   static get id () { return 'aws' }
+  static get isPayloadReporter () { return false }
 
   get serviceIdentifier () {
     const id = this.constructor.id.toLowerCase()
@@ -20,6 +22,14 @@ class BaseAwsSdkPlugin extends ClientPlugin {
     return id
   }
 
+  get cloudTaggingConfig () {
+    return this._tracerConfig.cloudPayloadTagging
+  }
+
+  get payloadTaggingRules () {
+    return this.cloudTaggingConfig.rules.aws?.[this.constructor.id]
+  }
+
   constructor (...args) {
     super(...args)
 
@@ -51,6 +61,12 @@ class BaseAwsSdkPlugin extends ClientPlugin {
 
       this.requestInject(span, request)
 
+      if (this.constructor.isPayloadReporter && this.cloudTaggingConfig.requestsEnabled) {
+        const maxDepth = this.cloudTaggingConfig.maxDepth
+        const requestTags = tagsFromRequest(this.payloadTaggingRules, request.params, { maxDepth })
+        span.addTags(requestTags)
+      }
+
       const store = storage.getStore()
 
       this.enter(span, store)
@@ -116,13 +132,30 @@ class BaseAwsSdkPlugin extends ClientPlugin {
     const params = response.request.params
     const operation = response.request.operation
     const extraTags = this.generateTags(params, operation, response) || {}
+
     const tags = Object.assign({
       'aws.response.request_id': response.requestId,
       'resource.name': operation,
       'span.kind': 'client'
     }, extraTags)
 
     span.addTags(tags)
+
+    if (this.constructor.isPayloadReporter && this.cloudTaggingConfig.responsesEnabled) {
+      const maxDepth = this.cloudTaggingConfig.maxDepth
+      const responseBody = this.extractResponseBody(response)
+      const responseTags = tagsFromResponse(this.payloadTaggingRules, responseBody, { maxDepth })
+      span.addTags(responseTags)
+    }
+  }
+
+  extractResponseBody (response) {
+    if (response.hasOwnProperty('data')) {
+      return response.data
+    }
+    return Object.fromEntries(
+      Object.entries(response).filter(([key]) => !['request', 'requestId', 'error', '$metadata'].includes(key))
+    )
   }
 
   generateTags () {

packages/datadog-plugin-aws-sdk/src/services/sns.js

@@ -7,6 +7,7 @@ const BaseAwsSdkPlugin = require('../base')
 class Sns extends BaseAwsSdkPlugin {
   static get id () { return 'sns' }
   static get peerServicePrecursors () { return ['topicname'] }
+  static get isPayloadReporter () { return true }
 
   generateTags (params, operation, response) {
     if (!params) return {}
@@ -20,6 +21,7 @@ class Sns extends BaseAwsSdkPlugin {
 
     // Get the topic name from the last part of the ARN
     const topicName = arnParts[arnParts.length - 1]
+
     return {
       'resource.name': `${operation} ${params.TopicArn || response.data.TopicArn}`,
       'aws.sns.topic_arn': TopicArn,