access log: add support for command formatter extensions

REPO_LAYOUT.md

@@ -89,6 +89,8 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE
     `Envoy::Extensions::ListenerFilters` namespace.
   * [filters/network/](/source/extensions/filters/network): L4 network filters which use the
     `Envoy::Extensions::NetworkFilters` namespace.
+  * [formatters](/source/extensions/formatters): Access log formatters which use the
+    `Envoy::Extensions::Formatters` namespace.
   * [grpc_credentials/](/source/extensions/grpc_credentials): Custom gRPC credentials which use the
     `Envoy::Extensions::GrpcCredentials` namespace.
   * [health_checker/](/source/extensions/health_checker): Custom health checkers which use the

api/envoy/config/core/v3/substitution_format_string.proto

@@ -3,7 +3,9 @@ syntax = "proto3";
 package envoy.config.core.v3;
 
 import "envoy/config/core/v3/base.proto";
+import "envoy/config/core/v3/extension.proto";
 
+import "google/protobuf/any.proto";
 import "google/protobuf/struct.proto";
 
 import "udpa/annotations/status.proto";
@@ -18,7 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 
 // Configuration to use multiple :ref:`command operators <config_access_log_command_operators>`
 // to generate a new string in either plain text or JSON format.
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message SubstitutionFormatString {
   oneof format {
     option (validate.required) = true;
@@ -103,4 +105,8 @@ message SubstitutionFormatString {
   //   content_type: "text/html; charset=UTF-8"
   //
   string content_type = 4;
+
+  // Specifies a collection of Formatter plugins that can be called from the access log configuration.
+  // See the formatters extensions documentation for details.
+  repeated TypedExtensionConfig formatters = 6;
 }

docs/root/extending/extending.rst

@@ -26,6 +26,7 @@ types including:
 * :ref:`Compression libraries <arch_overview_compression_libraries>`
 * :ref:`Bootstrap extensions <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.bootstrap_extensions>`
 * :ref:`Fatal actions <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.fatal_actions>`
+* :ref:`Formatters <config_access_log_command_operators>`
 
 As of this writing there is no high level extension developer documentation. The
 :repo:`existing extensions <source/extensions>` are a good way to learn what is possible.

docs/root/version_history/current.rst

@@ -19,6 +19,7 @@ Removed Config or Runtime
 
 New Features
 ------------
+* access log: added the :ref:`formatters <envoy_v3_api_field_config.core.v3.SubstitutionFormatString.formatters>` extension point for custom formatters (command operators).
 
 Deprecated
-----------
+----------

include/envoy/formatter/substitution_formatter.h

@@ -4,6 +4,7 @@
 #include <string>
 
 #include "envoy/common/pure.h"
+#include "envoy/config/typed_config.h"
 #include "envoy/http/header_map.h"
 #include "envoy/stream_info/stream_info.h"
 
@@ -78,5 +79,56 @@ class FormatterProvider {
 
 using FormatterProviderPtr = std::unique_ptr<FormatterProvider>;
 
+/**
+ * Interface for command parser.
+ * CommandParser returns a FormatterProviderPtr after successfully parsing an access log format
+ * token, nullptr otherwise.
+ */
+class CommandParser {
+public:
+  virtual ~CommandParser() = default;
+
+  /**
+   * Return a FormatterProviderPtr if this command is parsed from the token.
+   * @param token the token to parse
+   * @param pos current position in the entire format string
+   * @param command_end_position position at the end of the command token
+   *
+   * Given the following format line using an extension called %CMD()%:
+   *
+   * %CMD()% %START_TIME(%Y/%m/%d)% ...
+   *
+   * The call to parse() for that extension would look like this:
+   *
+   *  parse("CMD()", 1, 5)
+   *
+   * @return FormattterProviderPtr substitution provider for the parsed command
+   */
+  virtual FormatterProviderPtr parse(const std::string& token, size_t pos,
+                                     size_t command_end_position) const PURE;
+};
+
+using CommandParserPtr = std::unique_ptr<CommandParser>;
+
+/**
+ * Implemented by each custom CommandParser and registered via Registry::registerFactory()
+ * or the convenience class RegisterFactory.
+ */
+class CommandParserFactory : public Config::TypedFactory {
+public:
+  ~CommandParserFactory() override = default;
+
+  /**
+   * Creates a particular CommandParser implementation.
+   *
+   * @param config supplies the configuration for the command parser.
+   * @return CommandParserPtr the CommandParser which will be used in
+   * SubstitutionFormatParser::parse() when evaluating an access log format string.
+   */
+  virtual CommandParserPtr createCommandParserFromProto(const Protobuf::Message& config) PURE;
+
+  std::string category() const override { return "envoy.formatter"; }
+};
+
 } // namespace Formatter
 } // namespace Envoy

source/common/formatter/BUILD

@@ -35,6 +35,7 @@ envoy_cc_library(
     hdrs = ["substitution_format_string.h"],
     deps = [
         ":substitution_formatter_lib",
+        "//source/common/config:utility_lib",
         "//source/common/protobuf",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
     ],

source/common/formatter/substitution_format_string.cc

@@ -3,6 +3,7 @@
 #include "envoy/api/api.h"
 
 #include "common/config/datasource.h"
+#include "common/config/utility.h"
 #include "common/formatter/substitution_formatter.h"
 
 namespace Envoy {
@@ -16,14 +17,26 @@ SubstitutionFormatStringUtils::createJsonFormatter(const ProtobufWkt::Struct& st
 
 FormatterPtr SubstitutionFormatStringUtils::fromProtoConfig(
     const envoy::config::core::v3::SubstitutionFormatString& config, Api::Api& api) {
+  // Instantiate formatter extensions.
+  std::vector<CommandParserPtr> commands;
+  for (const auto& formatter : config.formatters()) {
+    auto* factory = Envoy::Config::Utility::getFactory<CommandParserFactory>(formatter);
+    if (!factory) {
+      throw EnvoyException(absl::StrCat("Formatter not found: ", formatter.name()));
+    }
+    auto parser = factory->createCommandParserFromProto(formatter.typed_config());
+    commands.push_back(std::move(parser));
+  }
+
   switch (config.format_case()) {
   case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormat:
-    return std::make_unique<FormatterImpl>(config.text_format(), config.omit_empty_values());
+    return std::make_unique<FormatterImpl>(config.text_format(), config.omit_empty_values(),
+                                           commands);
   case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kJsonFormat: {
     return createJsonFormatter(config.json_format(), true, config.omit_empty_values());
   case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormatSource:
     return std::make_unique<FormatterImpl>(
-        Config::DataSource::read(config.text_format_source(), true, api));
+        Config::DataSource::read(config.text_format_source(), true, api), false, commands);
   }
   default:
     NOT_REACHED_GCOVR_EXCL_LINE;