From 47139bafec41630d6f3ec80a64485bab614db1a5 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 19 Mar 2020 14:47:01 +0200 Subject: [PATCH 01/56] compressor: expose generic compressor filter to users Currently the generic HTTP compressor filter isn't exposed to users even though it's used internally by `envoy.filters.http.gzip` and can be used by external filter extensions. Expose the compressor's config API to users. For example the filter can be configured as follows: ... filter_chains: filters: - name: envoy.http_connection_manager config: http_filters: - name: envoy.filters.http.compressor config: disable_on_etag_header: true content_length: 100 content_type: - text/html - application/json compressor_library: name: envoy.filters.http.compressor.gzip config: memory_level: 3 window_bits: 10 compression_level: best compression_strategy: rle ... Multiple compressor filters using different compressor libraries, e.g. gzip and brotli, can be stacked in one filter chain. Signed-off-by: Dmitry Rozhkov --- CODEOWNERS | 2 + api/BUILD | 2 + .../filter/http/compressor/gzip/v2/BUILD | 9 + .../filter/http/compressor/gzip/v2/gzip.proto | 62 ++++ .../http/compressor/v2/compressor.proto | 28 +- .../filters/http/compressor/gzip/v3/BUILD | 12 + .../http/compressor/gzip/v3/gzip.proto | 66 ++++ .../http/compressor/v3/compressor.proto | 33 +- api/versioning/BUILD | 1 + docs/root/api-v2/config/filter/http/http.rst | 1 + docs/root/api-v3/config/filter/http/http.rst | 1 + .../http/http_filters/compressor_filter.rst | 80 ++++ .../http/http_filters/http_filters.rst | 1 + docs/root/intro/version_history.rst | 1 + .../filter/http/compressor/gzip/v2/BUILD | 9 + .../filter/http/compressor/gzip/v2/gzip.proto | 62 ++++ .../http/compressor/v2/compressor.proto | 28 +- .../filters/http/compressor/gzip/v3/BUILD | 12 + .../http/compressor/gzip/v3/gzip.proto | 66 ++++ .../http/compressor/v3/compressor.proto | 31 +- source/extensions/extensions_build_config.bzl | 2 + .../filters/http/common/compressor/BUILD | 1 + .../http/common/compressor/compressor.h | 7 +- .../extensions/filters/http/compressor/BUILD | 66 ++++ .../http/compressor/compressor_factory.h | 24 ++ .../http/compressor/compressor_filter.cc | 24 ++ .../http/compressor/compressor_filter.h | 35 ++ .../compressor/compressor_library_config.h | 22 ++ .../filters/http/compressor/config.cc | 46 +++ .../filters/http/compressor/config.h | 33 ++ .../filters/http/compressor/factory_base.h | 43 +++ .../filters/http/compressor/gzip/BUILD | 23 ++ .../filters/http/compressor/gzip/config.cc | 86 +++++ .../filters/http/compressor/gzip/config.h | 74 ++++ .../filters/http/well_known_names.h | 2 + test/extensions/filters/http/compressor/BUILD | 28 ++ .../compressor_filter_integration_test.cc | 341 ++++++++++++++++++ .../filters/http/compressor/gzip/BUILD | 34 ++ .../gzip/gzip_compressor_library_test.cc | 81 +++++ .../http/compressor/gzip/gzip_filter_test.cc | 169 +++++++++ 40 files changed, 1641 insertions(+), 7 deletions(-) create mode 100644 api/envoy/config/filter/http/compressor/gzip/v2/BUILD create mode 100644 api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto create mode 100644 api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD create mode 100644 api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto create mode 100644 docs/root/configuration/http/http_filters/compressor_filter.rst create mode 100644 generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD create mode 100644 generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto create mode 100644 generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD create mode 100644 generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto create mode 100644 source/extensions/filters/http/compressor/BUILD create mode 100644 source/extensions/filters/http/compressor/compressor_factory.h create mode 100644 source/extensions/filters/http/compressor/compressor_filter.cc create mode 100644 source/extensions/filters/http/compressor/compressor_filter.h create mode 100644 source/extensions/filters/http/compressor/compressor_library_config.h create mode 100644 source/extensions/filters/http/compressor/config.cc create mode 100644 source/extensions/filters/http/compressor/config.h create mode 100644 source/extensions/filters/http/compressor/factory_base.h create mode 100644 source/extensions/filters/http/compressor/gzip/BUILD create mode 100644 source/extensions/filters/http/compressor/gzip/config.cc create mode 100644 source/extensions/filters/http/compressor/gzip/config.h create mode 100644 test/extensions/filters/http/compressor/BUILD create mode 100644 test/extensions/filters/http/compressor/compressor_filter_integration_test.cc create mode 100644 test/extensions/filters/http/compressor/gzip/BUILD create mode 100644 test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc create mode 100644 test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 3580d19b7843..cb40d73e9194 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -20,6 +20,8 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/network/thrift_proxy @zuercher @brian-pane # compressor used by http compression filters /*/extensions/filters/http/common/compressor @gsagula @rojkov @dio +/*/extensions/filters/http/compressor @rojkov @dio +/*/extensions/filters/http/compressor/gzip @gsagula @rojkov @dio # jwt_authn http filter extension /*/extensions/filters/http/jwt_authn @qiwzhang @lizan # grpc_http1_reverse_bridge http filter extension diff --git a/api/BUILD b/api/BUILD index c8d845c6455c..804047f0ef04 100644 --- a/api/BUILD +++ b/api/BUILD @@ -32,6 +32,7 @@ proto_library( "//envoy/config/filter/http/aws_request_signing/v2alpha:pkg", "//envoy/config/filter/http/buffer/v2:pkg", "//envoy/config/filter/http/cache/v2alpha:pkg", + "//envoy/config/filter/http/compressor/gzip/v2:pkg", "//envoy/config/filter/http/compressor/v2:pkg", "//envoy/config/filter/http/cors/v2:pkg", "//envoy/config/filter/http/csrf/v2:pkg", @@ -165,6 +166,7 @@ proto_library( "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3alpha:pkg", + "//envoy/extensions/filters/http/compressor/gzip/v3:pkg", "//envoy/extensions/filters/http/compressor/v3:pkg", "//envoy/extensions/filters/http/cors/v3:pkg", "//envoy/extensions/filters/http/csrf/v3:pkg", diff --git a/api/envoy/config/filter/http/compressor/gzip/v2/BUILD b/api/envoy/config/filter/http/compressor/gzip/v2/BUILD new file mode 100644 index 000000000000..ef3541ebcb1d --- /dev/null +++ b/api/envoy/config/filter/http/compressor/gzip/v2/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto new file mode 100644 index 000000000000..ab6092e24aa4 --- /dev/null +++ b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -0,0 +1,62 @@ +syntax = "proto3"; + +package envoy.config.filter.http.compressor.gzip.v2; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.gzip.v2"; +option java_outer_classname = "GzipProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.filters.http.compressor.gzip.v3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Gzip] +// [#extension: envoy.filters.http.compressor.gzip] + +// [#next-free-field: 10] +message Gzip { + enum CompressionStrategy { + DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; + } + + message CompressionLevel { + enum Enum { + DEFAULT = 0; + BEST = 1; + SPEED = 2; + } + } + + // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values + // use more memory, but are faster and produce better compression results. The default value is 5. + google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; + + // A value used for selecting the zlib compression level. This setting will affect speed and + // amount of compression applied to the content. "BEST" provides higher compression at the cost of + // higher latency, "SPEED" provides lower compression with minimum impact on response time. + // "DEFAULT" provides an optimal result between speed and compression. This field will be set to + // "DEFAULT" if not specified. + CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + + // A value used for selecting the zlib compression strategy which is directly related to the + // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though + // there are situations which changing this parameter might produce better results. For example, + // run-length encoding (RLE) is typically used when the content is known for having sequences + // which same data occurs many consecutive times. For more information about each strategy, please + // refer to zlib manual. + CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + + // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 12 + // which will produce a 4096 bytes window. For more details about this parameter, please refer to + // zlib manual > deflateInit2. + google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; +} diff --git a/api/envoy/config/filter/http/compressor/v2/compressor.proto b/api/envoy/config/filter/http/compressor/v2/compressor.proto index 0d781a13672a..fbe220559bd5 100644 --- a/api/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/api/envoy/config/filter/http/compressor/v2/compressor.proto @@ -4,10 +4,13 @@ package envoy.config.filter.http.compressor.v2; import "envoy/api/v2/core/base.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.v2"; option java_outer_classname = "CompressorProto"; @@ -17,8 +20,10 @@ option (udpa.annotations.file_migrate).move_to_package = option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Compressor] +// Compressor :ref:`configuration overview `. +// [#extension: envoy.filters.http.compressor] -// [#next-free-field: 6] +// [#next-free-field: 7] message Compressor { // Minimum response length, in bytes, which will trigger compression. The default value is 30. google.protobuf.UInt32Value content_length = 1; @@ -45,4 +50,25 @@ message Compressor { // Runtime flag that controls whether the filter is enabled or not. If set to false, the // filter will operate as a pass-through filter. If not specified, defaults to enabled. api.v2.core.RuntimeFeatureFlag runtime_enabled = 5; + + // A compressor library to use for compression. By default + // :ref:`envoy.filters.http.compressor.gzip` + // is used. + // + // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" + // filter. + CompressorLibrary compressor_library = 6; +} + +message CompressorLibrary { + // The name of the compressor library to instantiate for the filter. + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Compressor library specific configuration. See the supported libraries for further + // documentation. + oneof config_type { + google.protobuf.Struct config = 2 [deprecated = true]; + + google.protobuf.Any typed_config = 3; + } } diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD b/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD new file mode 100644 index 000000000000..f7e871e7b656 --- /dev/null +++ b/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/filter/http/compressor/gzip/v2:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto new file mode 100644 index 000000000000..855c529f117a --- /dev/null +++ b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.compressor.gzip.v3; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.gzip.v3"; +option java_outer_classname = "GzipProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Gzip] +// [#extension: envoy.filters.http.compressor.gzip] + +// [#next-free-field: 10] +message Gzip { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.compressor.gzip.v2.Gzip"; + + enum CompressionStrategy { + DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; + } + + message CompressionLevel { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.compressor.gzip.v2.Gzip.CompressionLevel"; + + enum Enum { + DEFAULT = 0; + BEST = 1; + SPEED = 2; + } + } + + // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values + // use more memory, but are faster and produce better compression results. The default value is 5. + google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; + + // A value used for selecting the zlib compression level. This setting will affect speed and + // amount of compression applied to the content. "BEST" provides higher compression at the cost of + // higher latency, "SPEED" provides lower compression with minimum impact on response time. + // "DEFAULT" provides an optimal result between speed and compression. This field will be set to + // "DEFAULT" if not specified. + CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + + // A value used for selecting the zlib compression strategy which is directly related to the + // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though + // there are situations which changing this parameter might produce better results. For example, + // run-length encoding (RLE) is typically used when the content is known for having sequences + // which same data occurs many consecutive times. For more information about each strategy, please + // refer to zlib manual. + CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + + // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 12 + // which will produce a 4096 bytes window. For more details about this parameter, please refer to + // zlib manual > deflateInit2. + google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; +} diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index fabcc8dfead6..591193c896b0 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -4,10 +4,13 @@ package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.v3"; option java_outer_classname = "CompressorProto"; @@ -15,8 +18,10 @@ option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; // [#protodoc-title: Compressor] +// Compressor :ref:`configuration overview `. +// [#extension: envoy.filters.http.compressor] -// [#next-free-field: 6] +// [#next-free-field: 7] message Compressor { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.Compressor"; @@ -46,4 +51,30 @@ message Compressor { // Runtime flag that controls whether the filter is enabled or not. If set to false, the // filter will operate as a pass-through filter. If not specified, defaults to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; + + // A compressor library to use for compression. By default + // :ref:`envoy.filters.http.compressor.gzip` + // is used. + // + // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" + // filter. + CompressorLibrary compressor_library = 6; +} + +message CompressorLibrary { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.compressor.v2.CompressorLibrary"; + + reserved 2; + + reserved "config"; + + // The name of the compressor library to instantiate for the filter. + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Compressor library specific configuration. See the supported libraries for further + // documentation. + oneof config_type { + google.protobuf.Any typed_config = 3; + } } diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 9fb68272e125..300754166a9d 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -33,6 +33,7 @@ proto_library( "//envoy/config/filter/http/aws_request_signing/v2alpha:pkg", "//envoy/config/filter/http/buffer/v2:pkg", "//envoy/config/filter/http/cache/v2alpha:pkg", + "//envoy/config/filter/http/compressor/gzip/v2:pkg", "//envoy/config/filter/http/compressor/v2:pkg", "//envoy/config/filter/http/cors/v2:pkg", "//envoy/config/filter/http/csrf/v2:pkg", diff --git a/docs/root/api-v2/config/filter/http/http.rst b/docs/root/api-v2/config/filter/http/http.rst index 7746b0d72a5c..b54c56b28e41 100644 --- a/docs/root/api-v2/config/filter/http/http.rst +++ b/docs/root/api-v2/config/filter/http/http.rst @@ -9,3 +9,4 @@ HTTP filters */v2/* */v2alpha/* */v2alpha1/* + compressor/*/v2/* diff --git a/docs/root/api-v3/config/filter/http/http.rst b/docs/root/api-v3/config/filter/http/http.rst index 20f2c75664db..a5090c0634c6 100644 --- a/docs/root/api-v3/config/filter/http/http.rst +++ b/docs/root/api-v3/config/filter/http/http.rst @@ -7,3 +7,4 @@ HTTP filters */empty/* ../../../extensions/filters/http/*/v3/* + ../../../extensions/filters/http/compressor/*/v3/* diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst new file mode 100644 index 000000000000..2a129f613d0b --- /dev/null +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -0,0 +1,80 @@ +.. _config_http_filters_compressor: + +Compressor +========== +Compressor is an HTTP filter which enables Envoy to compress dispatched data +from an upstream service upon client request. Compression is useful in +situations where large payloads need to be transmitted without +compromising the response time. + +Configuration +------------- +* :ref:`v2 API reference ` +* This filter should be configured with the name *envoy.filters.http.compressor*. + +How it works +------------ +When compressor filter is enabled, request and response headers are inspected to +determine whether or not the content should be compressed. The content is +compressed and then sent to the client with the appropriate headers, if +response and request allow. + +Currently the filter supports :ref:`gzip compression` +only. Other compression libraries can be supported as extensions. + +By *default* compression will be *skipped* when: + +- A request does NOT contain *accept-encoding* header. +- A request includes *accept-encoding* header, but it does not contain "gzip" or "\*". +- A request includes *accept-encoding* with "gzip" or "\*" with the weight "q=0". Note + that the "gzip" will have a higher weight then "\*". For example, if *accept-encoding* + is "gzip;q=0,\*;q=1", the filter will not compress. But if the header is set to + "\*;q=0,gzip;q=1", the filter will compress. +- A request whose *accept-encoding* header includes any encoding type with a higher + weight than "gzip"'s given the corresponding compression filter is present in the chain. +- A response contains a *content-encoding* header. +- A response contains a *cache-control* header whose value includes "no-transform". +- A response contains a *transfer-encoding* header whose value includes "gzip". +- A response does not contain a *content-type* value that matches one of the selected + mime-types, which default to *application/javascript*, *application/json*, + *application/xhtml+xml*, *image/svg+xml*, *text/css*, *text/html*, *text/plain*, + *text/xml*. +- Neither *content-length* nor *transfer-encoding* headers are present in + the response. +- Response size is smaller than 30 bytes (only applicable when *transfer-encoding* + is not chunked). + +Please note that in case the filter is configured to use a compression library extension +other than gzip it looks for content encoding in the *accept-encoding* header provided by +the extension. + +When compression is *applied*: + +- The *content-length* is removed from response headers. +- Response headers contain "*transfer-encoding: chunked*" and do not contain + "*content-encoding*" header. +- The "*vary: accept-encoding*" header is inserted on every response. + +.. _compressor-statistics: + +Statistics +---------- + +Every configured Compressor filter has statistics rooted at ..* with the following: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + compressed, Counter, Number of requests compressed. + not_compressed, Counter, Number of requests not compressed. + no_accept_header, Counter, Number of requests with no accept header sent. + header_identity, Counter, Number of requests sent with "identity" set as the *accept-encoding*. + header_compressor_used, Counter, Number of requests sent with "gzip" set as the *accept-encoding*. + header_compressor_overshadowed, Counter, Number of requests skipped by this filter instance because they were handled by another filter in the same filter chain. + header_wildcard, Counter, Number of requests sent with "\*" set as the *accept-encoding*. + header_not_valid, Counter, Number of requests sent with a not valid *accept-encoding* header (aka "q=0" or an unsupported encoding type). + total_uncompressed_bytes, Counter, The total uncompressed bytes of all the requests that were marked for compression. + total_compressed_bytes, Counter, The total compressed bytes of all the requests that were marked for compression. + content_length_too_small, Counter, Number of requests that accepted gzip encoding but did not compress because the payload was too small. + not_compressed_etag, Counter, Number of requests that were not compressed due to the etag header. *disable_on_etag_header* must be turned on for this to happen. diff --git a/docs/root/configuration/http/http_filters/http_filters.rst b/docs/root/configuration/http/http_filters/http_filters.rst index aa435d6d5a9f..3d541aed13cf 100644 --- a/docs/root/configuration/http/http_filters/http_filters.rst +++ b/docs/root/configuration/http/http_filters/http_filters.rst @@ -10,6 +10,7 @@ HTTP filters aws_lambda_filter aws_request_signing_filter buffer_filter + compressor_filter cors_filter csrf_filter dynamic_forward_proxy_filter diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 47b378e4f6f0..3d44985e9eba 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -21,6 +21,7 @@ Version history `google.api.HttpBody `_. * grpc-stats: add options to limit which messages stats are created for. * config: added stat :ref:`update_time `. +* compressor: generic :ref:`compressor ` filter exposed to users. * datasource: added retry policy for remote async data source. * dns: the STRICT_DNS cluster now only resolves to 0 hosts if DNS resolution successfully returns 0 hosts. * dns: added support for :ref:`dns_failure_refresh_rate ` for the :ref:`dns cache ` to set the DNS refresh rate during failures. diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD new file mode 100644 index 000000000000..ef3541ebcb1d --- /dev/null +++ b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto new file mode 100644 index 000000000000..ab6092e24aa4 --- /dev/null +++ b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -0,0 +1,62 @@ +syntax = "proto3"; + +package envoy.config.filter.http.compressor.gzip.v2; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.gzip.v2"; +option java_outer_classname = "GzipProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.filters.http.compressor.gzip.v3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Gzip] +// [#extension: envoy.filters.http.compressor.gzip] + +// [#next-free-field: 10] +message Gzip { + enum CompressionStrategy { + DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; + } + + message CompressionLevel { + enum Enum { + DEFAULT = 0; + BEST = 1; + SPEED = 2; + } + } + + // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values + // use more memory, but are faster and produce better compression results. The default value is 5. + google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; + + // A value used for selecting the zlib compression level. This setting will affect speed and + // amount of compression applied to the content. "BEST" provides higher compression at the cost of + // higher latency, "SPEED" provides lower compression with minimum impact on response time. + // "DEFAULT" provides an optimal result between speed and compression. This field will be set to + // "DEFAULT" if not specified. + CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + + // A value used for selecting the zlib compression strategy which is directly related to the + // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though + // there are situations which changing this parameter might produce better results. For example, + // run-length encoding (RLE) is typically used when the content is known for having sequences + // which same data occurs many consecutive times. For more information about each strategy, please + // refer to zlib manual. + CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + + // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 12 + // which will produce a 4096 bytes window. For more details about this parameter, please refer to + // zlib manual > deflateInit2. + google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; +} diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto index 0d781a13672a..fbe220559bd5 100644 --- a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto @@ -4,10 +4,13 @@ package envoy.config.filter.http.compressor.v2; import "envoy/api/v2/core/base.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.v2"; option java_outer_classname = "CompressorProto"; @@ -17,8 +20,10 @@ option (udpa.annotations.file_migrate).move_to_package = option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Compressor] +// Compressor :ref:`configuration overview `. +// [#extension: envoy.filters.http.compressor] -// [#next-free-field: 6] +// [#next-free-field: 7] message Compressor { // Minimum response length, in bytes, which will trigger compression. The default value is 30. google.protobuf.UInt32Value content_length = 1; @@ -45,4 +50,25 @@ message Compressor { // Runtime flag that controls whether the filter is enabled or not. If set to false, the // filter will operate as a pass-through filter. If not specified, defaults to enabled. api.v2.core.RuntimeFeatureFlag runtime_enabled = 5; + + // A compressor library to use for compression. By default + // :ref:`envoy.filters.http.compressor.gzip` + // is used. + // + // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" + // filter. + CompressorLibrary compressor_library = 6; +} + +message CompressorLibrary { + // The name of the compressor library to instantiate for the filter. + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Compressor library specific configuration. See the supported libraries for further + // documentation. + oneof config_type { + google.protobuf.Struct config = 2 [deprecated = true]; + + google.protobuf.Any typed_config = 3; + } } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD new file mode 100644 index 000000000000..f7e871e7b656 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/filter/http/compressor/gzip/v2:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto new file mode 100644 index 000000000000..855c529f117a --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.compressor.gzip.v3; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.gzip.v3"; +option java_outer_classname = "GzipProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Gzip] +// [#extension: envoy.filters.http.compressor.gzip] + +// [#next-free-field: 10] +message Gzip { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.compressor.gzip.v2.Gzip"; + + enum CompressionStrategy { + DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; + } + + message CompressionLevel { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.compressor.gzip.v2.Gzip.CompressionLevel"; + + enum Enum { + DEFAULT = 0; + BEST = 1; + SPEED = 2; + } + } + + // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values + // use more memory, but are faster and produce better compression results. The default value is 5. + google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; + + // A value used for selecting the zlib compression level. This setting will affect speed and + // amount of compression applied to the content. "BEST" provides higher compression at the cost of + // higher latency, "SPEED" provides lower compression with minimum impact on response time. + // "DEFAULT" provides an optimal result between speed and compression. This field will be set to + // "DEFAULT" if not specified. + CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + + // A value used for selecting the zlib compression strategy which is directly related to the + // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though + // there are situations which changing this parameter might produce better results. For example, + // run-length encoding (RLE) is typically used when the content is known for having sequences + // which same data occurs many consecutive times. For more information about each strategy, please + // refer to zlib manual. + CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + + // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 12 + // which will produce a 4096 bytes window. For more details about this parameter, please refer to + // zlib manual > deflateInit2. + google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; +} diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index fabcc8dfead6..fc123541e692 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -4,10 +4,13 @@ package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.v3"; option java_outer_classname = "CompressorProto"; @@ -15,8 +18,10 @@ option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; // [#protodoc-title: Compressor] +// Compressor :ref:`configuration overview `. +// [#extension: envoy.filters.http.compressor] -// [#next-free-field: 6] +// [#next-free-field: 7] message Compressor { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.Compressor"; @@ -46,4 +51,28 @@ message Compressor { // Runtime flag that controls whether the filter is enabled or not. If set to false, the // filter will operate as a pass-through filter. If not specified, defaults to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; + + // A compressor library to use for compression. By default + // :ref:`envoy.filters.http.compressor.gzip` + // is used. + // + // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" + // filter. + CompressorLibrary compressor_library = 6; +} + +message CompressorLibrary { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.compressor.v2.CompressorLibrary"; + + // The name of the compressor library to instantiate for the filter. + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Compressor library specific configuration. See the supported libraries for further + // documentation. + oneof config_type { + google.protobuf.Struct hidden_envoy_deprecated_config = 2 [deprecated = true]; + + google.protobuf.Any typed_config = 3; + } } diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 3c85a8a61a6a..72733e6eccb4 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -38,6 +38,8 @@ EXTENSIONS = { "envoy.filters.http.aws_request_signing": "//source/extensions/filters/http/aws_request_signing:config", "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cache": "//source/extensions/filters/http/cache:config", + "envoy.filters.http.compressor": "//source/extensions/filters/http/compressor:config", + "envoy.filters.http.compressor.gzip": "//source/extensions/filters/http/compressor/gzip:config", "envoy.filters.http.cors": "//source/extensions/filters/http/cors:config", "envoy.filters.http.csrf": "//source/extensions/filters/http/csrf:config", "envoy.filters.http.dynamic_forward_proxy": "//source/extensions/filters/http/dynamic_forward_proxy:config", diff --git a/source/extensions/filters/http/common/compressor/BUILD b/source/extensions/filters/http/common/compressor/BUILD index 55e6a87aa2c7..60995f895b0f 100644 --- a/source/extensions/filters/http/common/compressor/BUILD +++ b/source/extensions/filters/http/common/compressor/BUILD @@ -8,6 +8,7 @@ load( envoy_package() +# TODO(rojkov): move this library to source/extensions/filters/http/compressor/. envoy_cc_library( name = "compressor_lib", srcs = ["compressor.cc"], diff --git a/source/extensions/filters/http/common/compressor/compressor.h b/source/extensions/filters/http/common/compressor/compressor.h index fc99ab517d0d..8972a2d0db3e 100644 --- a/source/extensions/filters/http/common/compressor/compressor.h +++ b/source/extensions/filters/http/common/compressor/compressor.h @@ -55,12 +55,14 @@ struct CompressorStats { ALL_COMPRESSOR_STATS(GENERATE_COUNTER_STRUCT) }; +// TODO(rojkov): merge this class with Compressor::CompressorFilterConfig when the filter +// `envoy.filters.http.gzip` is fully deprecated and dropped. class CompressorFilterConfig { public: CompressorFilterConfig() = delete; virtual ~CompressorFilterConfig() = default; - virtual std::unique_ptr makeCompressor() PURE; + virtual std::unique_ptr makeCompressor() PURE; bool enabled() const { return enabled_.enabled(); } const CompressorStats& stats() { return stats_; } @@ -69,7 +71,6 @@ class CompressorFilterConfig { bool removeAcceptEncodingHeader() const { return remove_accept_encoding_header_; } uint32_t minimumLength() const { return content_length_; } const std::string contentEncoding() const { return content_encoding_; }; - const std::map registeredCompressors() const; protected: CompressorFilterConfig( @@ -148,7 +149,7 @@ class CompressorFilter : public Http::PassThroughFilter { bool shouldCompress(const EncodingDecision& decision) const; bool skip_compression_; - std::unique_ptr compressor_; + std::unique_ptr compressor_; const CompressorFilterConfigSharedPtr config_; std::unique_ptr accept_encoding_; }; diff --git a/source/extensions/filters/http/compressor/BUILD b/source/extensions/filters/http/compressor/BUILD new file mode 100644 index 000000000000..7a516471f72f --- /dev/null +++ b/source/extensions/filters/http/compressor/BUILD @@ -0,0 +1,66 @@ +licenses(["notice"]) # Apache 2 + +# HTTP L7 filter that performs compression with configurable compression libraries +# Public docs: docs/root/configuration/http_filters/compressor_filter.rst + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "compressor_factory_interface", + hdrs = ["compressor_factory.h"], + deps = [ + "//include/envoy/compressor:compressor_interface", + ], +) + +envoy_cc_library( + name = "compressor_filter_lib", + srcs = ["compressor_filter.cc"], + hdrs = ["compressor_filter.h"], + deps = [ + ":compressor_factory_interface", + "//source/extensions/filters/http/common/compressor:compressor_lib", + "@envoy_api//envoy/extensions/filters/http/compressor/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "compressor_library_config_interface", + hdrs = ["compressor_library_config.h"], + deps = [ + "//include/envoy/config:typed_config_interface", + ], +) + +envoy_cc_library( + name = "factory_base_lib", + hdrs = ["factory_base.h"], + deps = [ + ":compressor_factory_interface", + ":compressor_library_config_interface", + "//include/envoy/server:filter_config_interface", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + security_posture = "robust_to_untrusted_downstream", + deps = [ + ":compressor_filter_lib", + ":compressor_library_config_interface", + "//source/common/config:utility_lib", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/common:factory_base_lib", + "//source/extensions/filters/http/compressor/gzip:config", + "@envoy_api//envoy/extensions/filters/http/compressor/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/http/compressor/compressor_factory.h b/source/extensions/filters/http/compressor/compressor_factory.h new file mode 100644 index 000000000000..481e5999671e --- /dev/null +++ b/source/extensions/filters/http/compressor/compressor_factory.h @@ -0,0 +1,24 @@ +#pragma once + +#include "envoy/compressor/compressor.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +class CompressorFactory { +public: + virtual ~CompressorFactory() = default; + + virtual std::unique_ptr createCompressor() PURE; + virtual const std::string& statsPrefix() const PURE; + virtual const std::string& contentEncoding() const PURE; +}; + +using CompressorFactoryPtr = std::unique_ptr; + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc new file mode 100644 index 000000000000..ef5436b76719 --- /dev/null +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -0,0 +1,24 @@ +#include "extensions/filters/http/compressor/compressor_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +CompressorFilterConfig::CompressorFilterConfig( + const envoy::extensions::filters::http::compressor::v3::Compressor& generic_compressor, + const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, + CompressorFactoryPtr compressor_factory) + : Common::Compressors::CompressorFilterConfig( + generic_compressor, stats_prefix + compressor_factory->statsPrefix(), scope, runtime, + compressor_factory->contentEncoding()), + compressor_factory_(std::move(compressor_factory)) {} + +std::unique_ptr CompressorFilterConfig::makeCompressor() { + return compressor_factory_->createCompressor(); +} + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/compressor_filter.h b/source/extensions/filters/http/compressor/compressor_filter.h new file mode 100644 index 000000000000..0ac1322ea972 --- /dev/null +++ b/source/extensions/filters/http/compressor/compressor_filter.h @@ -0,0 +1,35 @@ +#pragma once + +#include "envoy/extensions/filters/http/compressor/v3/compressor.pb.h" + +#include "extensions/filters/http/common/compressor/compressor.h" +#include "extensions/filters/http/compressor/compressor_factory.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +/** + * Configuration for the compressor filter. + */ +class CompressorFilterConfig : public Common::Compressors::CompressorFilterConfig { + // TODO(rojkov): move functionality of Common::Compressors::CompressorFilterConfig + // to this class when `envoy.filters.http.gzip` is fully deprecated and dropped. +public: + CompressorFilterConfig() = delete; + CompressorFilterConfig( + const envoy::extensions::filters::http::compressor::v3::Compressor& genereic_compressor, + const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, + CompressorFactoryPtr compressor_factory); + + std::unique_ptr makeCompressor() override; + +private: + const CompressorFactoryPtr compressor_factory_; +}; + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/compressor_library_config.h b/source/extensions/filters/http/compressor/compressor_library_config.h new file mode 100644 index 000000000000..1b2b508acb80 --- /dev/null +++ b/source/extensions/filters/http/compressor/compressor_library_config.h @@ -0,0 +1,22 @@ +#pragma once + +#include "envoy/config/typed_config.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +class NamedCompressorLibraryConfigFactory : public Config::TypedFactory { +public: + ~NamedCompressorLibraryConfigFactory() override = default; + + virtual CompressorFactoryPtr + createCompressorLibraryFromProto(const Protobuf::Message& config, + Server::Configuration::FactoryContext& context) PURE; +}; + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc new file mode 100644 index 000000000000..5d22f8a91171 --- /dev/null +++ b/source/extensions/filters/http/compressor/config.cc @@ -0,0 +1,46 @@ +#include "extensions/filters/http/compressor/config.h" + +#include "common/config/utility.h" + +#include "extensions/filters/http/compressor/compressor_filter.h" +#include "extensions/filters/http/compressor/compressor_library_config.h" +#include "extensions/filters/http/compressor/gzip/config.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::compressor::v3::Compressor& proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + CompressorFactoryPtr compressor_factory; + + if (proto_config.has_compressor_library()) { + auto& config_factory = Config::Utility::getAndCheckFactory( + proto_config.compressor_library()); + ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( + proto_config.compressor_library(), context.messageValidationVisitor(), config_factory); + compressor_factory = config_factory.createCompressorLibraryFromProto(*message, context); + } else { + // By default use gzip compression. + compressor_factory = std::make_unique( + envoy::extensions::filters::http::compressor::gzip::v3::Gzip()); + } + Common::Compressors::CompressorFilterConfigSharedPtr config = + std::make_shared(proto_config, stats_prefix, context.scope(), + context.runtime(), std::move(compressor_factory)); + return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_shared(config)); + }; +} + +/** + * Static registration for the compressor filter. @see NamedHttpFilterConfigFactory. + */ +REGISTER_FACTORY(CompressorFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/config.h b/source/extensions/filters/http/compressor/config.h new file mode 100644 index 000000000000..50127769f3b4 --- /dev/null +++ b/source/extensions/filters/http/compressor/config.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/extensions/filters/http/compressor/v3/compressor.pb.h" +#include "envoy/extensions/filters/http/compressor/v3/compressor.pb.validate.h" + +#include "extensions/filters/http/common/factory_base.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +/** + * Config registration for the compressor filter. @see NamedHttpFilterConfigFactory. + */ +class CompressorFilterFactory + : public Common::FactoryBase { +public: + CompressorFilterFactory() : FactoryBase(HttpFilterNames::get().Compressor) {} + +private: + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::compressor::v3::Compressor& config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +DECLARE_FACTORY(CompressorFilterFactory); + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/factory_base.h b/source/extensions/filters/http/compressor/factory_base.h new file mode 100644 index 000000000000..ce564006c708 --- /dev/null +++ b/source/extensions/filters/http/compressor/factory_base.h @@ -0,0 +1,43 @@ +#pragma once + +#include "envoy/server/filter_config.h" + +#include "extensions/filters/http/compressor/compressor_factory.h" +#include "extensions/filters/http/compressor/compressor_library_config.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +template +class CompressorLibraryFactoryBase : public NamedCompressorLibraryConfigFactory { +public: + CompressorFactoryPtr + createCompressorLibraryFromProto(const Protobuf::Message& proto_config, + Server::Configuration::FactoryContext& context) override { + return createCompressorLibraryFromProtoTyped( + MessageUtil::downcastAndValidate(proto_config, + context.messageValidationVisitor())); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string category() const override { return "envoy.filters.http.compressor"; } + + std::string name() const override { return name_; } + +protected: + CompressorLibraryFactoryBase(const std::string& name) : name_(name) {} + +private: + virtual CompressorFactoryPtr createCompressorLibraryFromProtoTyped(const ConfigProto&) PURE; + const std::string name_; +}; + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/gzip/BUILD b/source/extensions/filters/http/compressor/gzip/BUILD new file mode 100644 index 000000000000..2c22ff68d107 --- /dev/null +++ b/source/extensions/filters/http/compressor/gzip/BUILD @@ -0,0 +1,23 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_package", +) + +envoy_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + security_posture = "robust_to_untrusted_downstream", + deps = [ + "//source/common/compressor:compressor_lib", + "//source/common/http:headers_lib", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/compressor:factory_base_lib", + "@envoy_api//envoy/extensions/filters/http/compressor/gzip/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/http/compressor/gzip/config.cc b/source/extensions/filters/http/compressor/gzip/config.cc new file mode 100644 index 000000000000..c47d33f13390 --- /dev/null +++ b/source/extensions/filters/http/compressor/gzip/config.cc @@ -0,0 +1,86 @@ +#include "extensions/filters/http/compressor/gzip/config.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { +namespace Gzip { + +namespace { +// Default zlib memory level. +const uint64_t DefaultMemoryLevel = 5; + +// Default and maximum compression window size. +const uint64_t DefaultWindowBits = 12; + +// When summed to window bits, this sets a gzip header and trailer around the compressed data. +const uint64_t GzipHeaderValue = 16; + +} // namespace + +GzipCompressorFactory::GzipCompressorFactory( + const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& gzip) + : compression_level_(compressionLevelEnum(gzip.compression_level())), + compression_strategy_(compressionStrategyEnum(gzip.compression_strategy())), + memory_level_(memoryLevelUint(gzip.memory_level().value())), + window_bits_(windowBitsUint(gzip.window_bits().value())) {} + +Envoy::Compressor::ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::compressionLevelEnum( + envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::Enum + compression_level) { + switch (compression_level) { + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::BEST: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::SPEED: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Speed; + default: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard; + } +} + +Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy +GzipCompressorFactory::compressionStrategyEnum( + envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionStrategy + compression_strategy) { + switch (compression_strategy) { + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::RLE: + return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::FILTERED: + return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::HUFFMAN: + return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman; + default: + return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard; + } +} + +uint64_t GzipCompressorFactory::memoryLevelUint(Protobuf::uint32 level) { + return level > 0 ? level : DefaultMemoryLevel; +} + +uint64_t GzipCompressorFactory::windowBitsUint(Protobuf::uint32 window_bits) { + return (window_bits > 0 ? window_bits : DefaultWindowBits) | GzipHeaderValue; +} + +std::unique_ptr GzipCompressorFactory::createCompressor() { + auto compressor = std::make_unique(); + compressor->init(compression_level_, compression_strategy_, window_bits_, memory_level_); + return compressor; +} + +Compressor::CompressorFactoryPtr +GzipCompressorLibraryFactory::createCompressorLibraryFromProtoTyped( + const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& proto_config) { + return std::make_unique(proto_config); +} + +/** + * Static registration for the gzip filter. @see NamedCompressorLibraryConfigFactory. + */ +REGISTER_FACTORY(GzipCompressorLibraryFactory, NamedCompressorLibraryConfigFactory); + +} // namespace Gzip +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/compressor/gzip/config.h b/source/extensions/filters/http/compressor/gzip/config.h new file mode 100644 index 000000000000..02a925a8dd9e --- /dev/null +++ b/source/extensions/filters/http/compressor/gzip/config.h @@ -0,0 +1,74 @@ +#pragma once + +#include "envoy/extensions/filters/http/compressor/gzip/v3/gzip.pb.h" +#include "envoy/extensions/filters/http/compressor/gzip/v3/gzip.pb.validate.h" + +#include "common/compressor/zlib_compressor_impl.h" +#include "common/http/headers.h" + +#include "extensions/filters/http/compressor/factory_base.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { +namespace Gzip { + +namespace { + +const std::string& gzipStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "gzip."); } +const std::string& gzipExtensionName() { + CONSTRUCT_ON_FIRST_USE(std::string, "envoy.filters.http.compressor.gzip"); +} + +} // namespace + +class GzipCompressorFactory : public Compressor::CompressorFactory { +public: + GzipCompressorFactory(const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& gzip); + + std::unique_ptr createCompressor() override; + const std::string& statsPrefix() const override { return gzipStatsPrefix(); } + const std::string& contentEncoding() const override { + return Http::Headers::get().ContentEncodingValues.Gzip; + } + +private: + friend class GzipTest; + + static Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compressionLevelEnum( + envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::Enum + compression_level); + static Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategyEnum( + envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionStrategy + compression_strategy); + + static uint64_t memoryLevelUint(Protobuf::uint32 level); + static uint64_t windowBitsUint(Protobuf::uint32 window_bits); + + Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compression_level_; + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compression_strategy_; + + int32_t memory_level_; + int32_t window_bits_; +}; + +class GzipCompressorLibraryFactory + : public CompressorLibraryFactoryBase< + envoy::extensions::filters::http::compressor::gzip::v3::Gzip> { +public: + GzipCompressorLibraryFactory() : CompressorLibraryFactoryBase(gzipExtensionName()) {} + +private: + Compressor::CompressorFactoryPtr createCompressorLibraryFromProtoTyped( + const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& config) override; +}; + +DECLARE_FACTORY(GzipCompressorLibraryFactory); + +} // namespace Gzip +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/well_known_names.h b/source/extensions/filters/http/well_known_names.h index 68bc5c361be4..afa9981a7510 100644 --- a/source/extensions/filters/http/well_known_names.h +++ b/source/extensions/filters/http/well_known_names.h @@ -16,6 +16,8 @@ class HttpFilterNameValues { const std::string Buffer = "envoy.filters.http.buffer"; // Cache filter const std::string Cache = "envoy.filters.http.cache"; + // Compressor filter + const std::string Compressor = "envoy.filters.http.compressor"; // CORS filter const std::string Cors = "envoy.filters.http.cors"; // CSRF filter diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD new file mode 100644 index 000000000000..7e84ab2a8927 --- /dev/null +++ b/test/extensions/filters/http/compressor/BUILD @@ -0,0 +1,28 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "compressor_filter_integration_test", + srcs = [ + "compressor_filter_integration_test.cc", + ], + extension_name = "envoy.filters.http.compressor", + deps = [ + "//source/common/decompressor:decompressor_lib", + "//source/extensions/filters/http/compressor:config", + "//source/extensions/filters/http/compressor/gzip:config", + "//test/integration:http_integration_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc new file mode 100644 index 000000000000..9b611f891fd2 --- /dev/null +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -0,0 +1,341 @@ +#include "envoy/event/timer.h" + +#include "common/decompressor/zlib_decompressor_impl.h" + +#include "test/integration/http_integration.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +class GzipIntegrationTest : public testing::TestWithParam, + public Event::SimulatedTimeSystem, + public HttpIntegrationTest { +public: + GzipIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + + void SetUp() override { decompressor_.init(window_bits); } + void TearDown() override { cleanupUpstreamAndDownstream(); } + + void initializeFilter(const std::string& config) { + config_helper_.addFilter(config); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + } + + void doRequestAndCompression(Http::TestHeaderMapImpl&& request_headers, + Http::TestHeaderMapImpl&& response_headers) { + uint64_t content_length; + ASSERT_TRUE(absl::SimpleAtoi(response_headers.get_("content-length"), &content_length)); + const Buffer::OwnedImpl expected_response{std::string(content_length, 'a')}; + auto response = + sendRequestAndWaitForResponse(request_headers, 0, response_headers, content_length); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + ASSERT_TRUE(response->headers().ContentEncoding() != nullptr); + EXPECT_EQ(Http::Headers::get().ContentEncodingValues.Gzip, + response->headers().ContentEncoding()->value().getStringView()); + ASSERT_TRUE(response->headers().TransferEncoding() != nullptr); + EXPECT_EQ(Http::Headers::get().TransferEncodingValues.Chunked, + response->headers().TransferEncoding()->value().getStringView()); + + Buffer::OwnedImpl decompressed_response{}; + const Buffer::OwnedImpl compressed_response{response->body()}; + decompressor_.decompress(compressed_response, decompressed_response); + ASSERT_EQ(content_length, decompressed_response.length()); + EXPECT_TRUE(TestUtility::buffersEqual(expected_response, decompressed_response)); + } + + void doRequestAndNoCompression(Http::TestHeaderMapImpl&& request_headers, + Http::TestHeaderMapImpl&& response_headers) { + uint64_t content_length; + ASSERT_TRUE(absl::SimpleAtoi(response_headers.get_("content-length"), &content_length)); + auto response = + sendRequestAndWaitForResponse(request_headers, 0, response_headers, content_length); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + ASSERT_TRUE(response->headers().ContentEncoding() == nullptr); + ASSERT_EQ(content_length, response->body().size()); + EXPECT_EQ(response->body(), std::string(content_length, 'a')); + } + + const std::string deprecated_full_config{R"EOF( + name: envoy.filters.http.compressor + config: + disable_on_etag_header: true + content_length: 100 + content_type: + - text/html + - application/json + compressor_library: + name: envoy.filters.http.compressor.gzip + config: + memory_level: 3 + window_bits: 10 + compression_level: best + compression_strategy: rle + )EOF"}; + + const std::string full_config{R"EOF( + name: compressor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor + disable_on_etag_header: true + content_length: 100 + content_type: + - text/html + - application/json + compressor_library: + name: gzip + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip + memory_level: 3 + window_bits: 10 + compression_level: best + compression_strategy: rle + )EOF"}; + + const std::string default_config{"name: envoy.filters.http.compressor"}; + + const uint64_t window_bits{15 | 16}; + + Decompressor::ZlibDecompressorImpl decompressor_{}; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, GzipIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +/** + * Exercises gzip compression with default configuration. + */ +TEST_P(GzipIntegrationTest, AcceptanceDefaultConfigTest) { + initializeFilter(default_config); + doRequestAndCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}, + Http::TestResponseHeaderMapImpl{{":status", "200"}, + {"content-length", "4400"}, + {"content-type", "text/xml"}}); +} + +/** + * Exercises gzip compression with deprecated full configuration. + */ +TEST_P(GzipIntegrationTest, AcceptanceDeprecatedFullConfigTest) { + initializeFilter(deprecated_full_config); + doRequestAndCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}, + Http::TestResponseHeaderMapImpl{{":status", "200"}, + {"content-length", "4400"}, + {"content-type", "application/json"}}); +} + +/** + * Exercises gzip compression with full configuration. + */ +TEST_P(GzipIntegrationTest, AcceptanceFullConfigTest) { + initializeFilter(full_config); + doRequestAndCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}, + Http::TestResponseHeaderMapImpl{{":status", "200"}, + {"content-length", "4400"}, + {"content-type", "application/json"}}); +} + +/** + * Exercises filter when client request contains 'identity' type. + */ +TEST_P(GzipIntegrationTest, IdentityAcceptEncoding) { + initializeFilter(default_config); + doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "identity"}}, + Http::TestResponseHeaderMapImpl{{":status", "200"}, + {"content-length", "128"}, + {"content-type", "text/plain"}}); +} + +/** + * Exercises filter when client request contains unsupported 'accept-encoding' type. + */ +TEST_P(GzipIntegrationTest, NotSupportedAcceptEncoding) { + initializeFilter(default_config); + doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, br"}}, + Http::TestResponseHeaderMapImpl{{":status", "200"}, + {"content-length", "128"}, + {"content-type", "text/plain"}}); +} + +/** + * Exercises filter when upstream response is already encoded. + */ +TEST_P(GzipIntegrationTest, UpstreamResponseAlreadyEncoded) { + initializeFilter(default_config); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}; + + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}, + {"content-encoding", "br"}, + {"content-length", "128"}, + {"content-type", "application/json"}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, response_headers, 128); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + ASSERT_EQ("br", response->headers().ContentEncoding()->value().getStringView()); + EXPECT_EQ(128U, response->body().size()); +} + +/** + * Exercises filter when upstream responds with content length below the default threshold. + */ +TEST_P(GzipIntegrationTest, NotEnoughContentLength) { + initializeFilter(default_config); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}; + + Http::TestResponseHeaderMapImpl response_headers{ + {":status", "200"}, {"content-length", "10"}, {"content-type", "application/json"}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, response_headers, 10); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + ASSERT_TRUE(response->headers().ContentEncoding() == nullptr); + EXPECT_EQ(10U, response->body().size()); +} + +/** + * Exercises filter when response from upstream service is empty. + */ +TEST_P(GzipIntegrationTest, EmptyResponse) { + initializeFilter(default_config); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}; + + Http::TestResponseHeaderMapImpl response_headers{{":status", "204"}, {"content-length", "0"}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, response_headers, 0); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("204", response->headers().Status()->value().getStringView()); + ASSERT_TRUE(response->headers().ContentEncoding() == nullptr); + EXPECT_EQ(0U, response->body().size()); +} + +/** + * Exercises filter when upstream responds with restricted content-type value. + */ +TEST_P(GzipIntegrationTest, SkipOnContentType) { + initializeFilter(full_config); + doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}, + Http::TestResponseHeaderMapImpl{{":status", "200"}, + {"content-length", "128"}, + {"content-type", "application/xml"}}); +} + +/** + * Exercises filter when upstream responds with restricted cache-control value. + */ +TEST_P(GzipIntegrationTest, SkipOnCacheControl) { + initializeFilter(full_config); + doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}, + Http::TestResponseHeaderMapImpl{{":status", "200"}, + {"content-length", "128"}, + {"cache-control", "no-transform"}, + {"content-type", "application/json"}}); +} + +/** + * Exercises gzip compression when upstream returns a chunked response. + */ +TEST_P(GzipIntegrationTest, AcceptanceFullConfigChunkedResponse) { + initializeFilter(full_config); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}; + + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}, + {"content-type", "application/json"}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, response_headers, 1024); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + ASSERT_EQ("gzip", response->headers().ContentEncoding()->value().getStringView()); + ASSERT_EQ("chunked", response->headers().TransferEncoding()->value().getStringView()); +} + +/** + * Verify Vary header values are preserved. + */ +TEST_P(GzipIntegrationTest, AcceptanceFullConfigVeryHeader) { + initializeFilter(default_config); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"accept-encoding", "deflate, gzip"}}; + + Http::TestResponseHeaderMapImpl response_headers{ + {":status", "200"}, {"content-type", "application/json"}, {"vary", "Cookie"}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, response_headers, 1024); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(0U, upstream_request_->bodyLength()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + ASSERT_EQ("gzip", response->headers().ContentEncoding()->value().getStringView()); + ASSERT_EQ("Cookie, Accept-Encoding", response->headers().Vary()->value().getStringView()); +} +} // namespace Envoy diff --git a/test/extensions/filters/http/compressor/gzip/BUILD b/test/extensions/filters/http/compressor/gzip/BUILD new file mode 100644 index 000000000000..c666743a44b9 --- /dev/null +++ b/test/extensions/filters/http/compressor/gzip/BUILD @@ -0,0 +1,34 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "gzip_compressor_library_test", + srcs = ["gzip_compressor_library_test.cc"], + extension_name = "envoy.filters.http.compressor.gzip", + deps = [ + "//source/extensions/filters/http/compressor/gzip:config", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "gzip_filter_test", + srcs = ["gzip_filter_test.cc"], + extension_name = "envoy.filters.http.compressor.gzip", + deps = [ + "//source/common/decompressor:decompressor_lib", + "//source/extensions/filters/http/compressor:compressor_filter_lib", + "//source/extensions/filters/http/compressor/gzip:config", + "//test/mocks/http:http_mocks", + ], +) diff --git a/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc b/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc new file mode 100644 index 000000000000..5a2d904917a6 --- /dev/null +++ b/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc @@ -0,0 +1,81 @@ +#include "extensions/filters/http/compressor/gzip/config.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { +namespace Gzip { + +class GzipTest : public testing::Test { +protected: + void SetUp() override { setUpGzip("{}"); } + + // GzipTest Helpers + void setUpGzip(std::string&& json) { + envoy::extensions::filters::http::compressor::gzip::v3::Gzip gzip; + TestUtility::loadFromJson(json, gzip); + factory_ = std::make_unique(gzip); + } + + Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compressionLevel() const { + return factory_->compression_level_; + } + + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategy() const { + return factory_->compression_strategy_; + } + + int32_t memoryLevel() const { return factory_->memory_level_; } + int32_t windowBits() const { return factory_->window_bits_; } + + void expectValidCompressionStrategyAndLevel( + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy strategy, + absl::string_view strategy_name, + Envoy::Compressor::ZlibCompressorImpl::CompressionLevel level, absl::string_view level_name) { + setUpGzip(fmt::format( + R"EOF({{"compression_strategy": "{}", "compression_level": "{}", "memory_level": 6, "window_bits": 27}})EOF", + strategy_name, level_name)); + EXPECT_EQ(strategy, compressionStrategy()); + EXPECT_EQ(level, compressionLevel()); + EXPECT_EQ(6, memoryLevel()); + EXPECT_EQ(27, windowBits()); + } + + std::unique_ptr factory_; +}; + +// Default config values. +TEST_F(GzipTest, DefaultConfigValues) { + EXPECT_EQ(5, memoryLevel()); + EXPECT_EQ(28, windowBits()); + EXPECT_EQ(Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + compressionStrategy()); + EXPECT_EQ(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, compressionLevel()); +} + +TEST_F(GzipTest, AvailableCombinationCompressionStrategyAndLevelConfig) { + expectValidCompressionStrategyAndLevel( + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, "FILTERED", + Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST"); + expectValidCompressionStrategyAndLevel( + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, "HUFFMAN", + Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST"); + expectValidCompressionStrategyAndLevel( + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, "RLE", + Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Speed, "SPEED"); + expectValidCompressionStrategyAndLevel( + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, "DEFAULT", + Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, "DEFAULT"); +} + +} // namespace Gzip +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc new file mode 100644 index 000000000000..2285d75f8b00 --- /dev/null +++ b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc @@ -0,0 +1,169 @@ +#include "common/common/hex.h" +#include "common/decompressor/zlib_decompressor_impl.h" + +#include "extensions/filters/http/compressor/compressor_filter.h" +#include "extensions/filters/http/compressor/gzip/config.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/stats/mocks.h" + +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { + +class GzipFilterTest : public testing::Test { +protected: + void SetUp() override { + envoy::extensions::filters::http::compressor::v3::Compressor compressor; + envoy::extensions::filters::http::compressor::gzip::v3::Gzip gzip; + CompressorFactoryPtr gzip_factory = std::make_unique(gzip); + config_.reset( + new CompressorFilterConfig(compressor, "test.", stats_, runtime_, std::move(gzip_factory))); + filter_ = std::make_unique(config_); + filter_->setEncoderFilterCallbacks(encoder_callbacks_); + filter_->setDecoderFilterCallbacks(decoder_callbacks_); + decompressor_.init(31); + } + + // GzipFilterTest Helpers + void verifyCompressedData(const uint32_t content_length) { + // This makes sure we have a finished buffer before sending it to the client. + expectValidFinishedBuffer(content_length); + decompressor_.decompress(data_, decompressed_data_); + const std::string uncompressed_str{decompressed_data_.toString()}; + ASSERT_EQ(expected_str_.length(), uncompressed_str.length()); + EXPECT_EQ(expected_str_, uncompressed_str); + EXPECT_EQ(expected_str_.length(), stats_.counter("test.gzip.total_uncompressed_bytes").value()); + EXPECT_EQ(data_.length(), stats_.counter("test.gzip.total_compressed_bytes").value()); + } + + void feedBuffer(uint64_t size) { + TestUtility::feedBufferWithRandomCharacters(data_, size); + expected_str_ += data_.toString(); + } + + void drainBuffer() { + const uint64_t data_len = data_.length(); + data_.drain(data_len); + } + + void doRequest(Http::TestRequestHeaderMapImpl&& headers, bool end_stream) { + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, end_stream)); + } + + void doResponseCompression(Http::TestResponseHeaderMapImpl&& headers, bool with_trailers) { + uint64_t content_length; + ASSERT_TRUE(absl::SimpleAtoi(headers.get_("content-length"), &content_length)); + feedBuffer(content_length); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); + EXPECT_EQ("", headers.get_("content-length")); + EXPECT_EQ(Http::Headers::get().ContentEncodingValues.Gzip, headers.get_("content-encoding")); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, !with_trailers)); + if (with_trailers) { + Buffer::OwnedImpl trailers_buffer; + EXPECT_CALL(encoder_callbacks_, addEncodedData(_, true)) + .WillOnce(Invoke([&](Buffer::Instance& data, bool) { data_.move(data); })); + Http::TestResponseTrailerMapImpl trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(trailers)); + } + verifyCompressedData(content_length); + drainBuffer(); + EXPECT_EQ(1U, stats_.counter("test.gzip.compressed").value()); + } + + void expectValidFinishedBuffer(const uint32_t content_length) { + Buffer::RawSliceVector compressed_slices = data_.getRawSlices(); + const uint64_t num_comp_slices = compressed_slices.size(); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_bytes_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + + // A valid finished compressed buffer should have trailer with input size in it, i.e. equals to + // the value of content_length. + expectEqualInputSize(footer_bytes_str, content_length); + } + + void expectEqualInputSize(const std::string& footer_bytes, const uint32_t input_size) { + const std::string size_bytes = footer_bytes.substr(footer_bytes.size() - 8, 8); + uint64_t size; + StringUtil::atoull(size_bytes.c_str(), size, 16); + EXPECT_EQ(TestUtility::flipOrder(size), input_size); + } + + void doResponseNoCompression(Http::TestResponseHeaderMapImpl&& headers) { + uint64_t content_length; + ASSERT_TRUE(absl::SimpleAtoi(headers.get_("content-length"), &content_length)); + feedBuffer(content_length); + Http::TestResponseHeaderMapImpl continue_headers; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->encode100ContinueHeaders(continue_headers)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); + EXPECT_EQ("", headers.get_("content-encoding")); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, false)); + Http::TestResponseTrailerMapImpl trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(trailers)); + EXPECT_EQ(1, stats_.counter("test.gzip.not_compressed").value()); + } + + std::shared_ptr config_; + std::unique_ptr filter_; + Buffer::OwnedImpl data_; + Decompressor::ZlibDecompressorImpl decompressor_; + Buffer::OwnedImpl decompressed_data_; + std::string expected_str_; + Stats::IsolatedStoreImpl stats_; + NiceMock runtime_; + NiceMock encoder_callbacks_; + NiceMock decoder_callbacks_; +}; + +// Acceptance Testing with default configuration. +TEST_F(GzipFilterTest, AcceptanceGzipEncoding) { + doRequest({{":method", "get"}, {"accept-encoding", "deflate, gzip"}}, false); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + Buffer::OwnedImpl data("hello"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); + Http::TestRequestTrailerMapImpl trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); + doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); +} + +TEST_F(GzipFilterTest, AcceptanceGzipEncodingWithTrailers) { + doRequest({{":method", "get"}, {"accept-encoding", "deflate, gzip"}}, false); + Buffer::OwnedImpl data("hello"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); + Http::TestRequestTrailerMapImpl trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); + doResponseCompression({{":method", "get"}, {"content-length", "256"}}, true); +} + +// Verifies that compression is skipped when accept-encoding header is not allowed. +TEST_F(GzipFilterTest, AcceptEncodingNoCompression) { + doRequest({{":method", "get"}, {"accept-encoding", "gzip;q=0, deflate"}}, true); + doResponseNoCompression({{":method", "get"}, {"content-length", "256"}}); +} + +// Verifies that compression is NOT skipped when accept-encoding header is allowed. +TEST_F(GzipFilterTest, AcceptEncodingCompression) { + doRequest({{":method", "get"}, {"accept-encoding", "gzip, deflate"}}, true); + doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); +} + +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy From 4bc966060ec2cbf82670df82c1023d24a727c2de Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Fri, 27 Mar 2020 14:58:50 +0200 Subject: [PATCH 02/56] fix naming conflict in compressor_filter_integration_test.cc Signed-off-by: Dmitry Rozhkov --- .../compressor_filter_integration_test.cc | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 9b611f891fd2..6db776fdb9a1 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -10,11 +10,11 @@ namespace Envoy { -class GzipIntegrationTest : public testing::TestWithParam, +class CompressorIntegrationTest : public testing::TestWithParam, public Event::SimulatedTimeSystem, public HttpIntegrationTest { public: - GzipIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + CompressorIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} void SetUp() override { decompressor_.init(window_bits); } void TearDown() override { cleanupUpstreamAndDownstream(); } @@ -108,14 +108,14 @@ class GzipIntegrationTest : public testing::TestWithParam Date: Fri, 27 Mar 2020 16:15:03 +0200 Subject: [PATCH 03/56] fix clang-tidy findings Signed-off-by: Dmitry Rozhkov --- source/extensions/filters/http/compressor/BUILD | 2 ++ .../filters/http/compressor/compressor_library_config.h | 3 +++ .../http/compressor/compressor_filter_integration_test.cc | 4 ++-- .../http/compressor/gzip/gzip_compressor_library_test.cc | 2 -- .../filters/http/compressor/gzip/gzip_filter_test.cc | 8 ++++---- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/source/extensions/filters/http/compressor/BUILD b/source/extensions/filters/http/compressor/BUILD index 7a516471f72f..3230e998f159 100644 --- a/source/extensions/filters/http/compressor/BUILD +++ b/source/extensions/filters/http/compressor/BUILD @@ -35,7 +35,9 @@ envoy_cc_library( name = "compressor_library_config_interface", hdrs = ["compressor_library_config.h"], deps = [ + ":compressor_factory_interface", "//include/envoy/config:typed_config_interface", + "//include/envoy/server:filter_config_interface", ], ) diff --git a/source/extensions/filters/http/compressor/compressor_library_config.h b/source/extensions/filters/http/compressor/compressor_library_config.h index 1b2b508acb80..a4dcdaad2659 100644 --- a/source/extensions/filters/http/compressor/compressor_library_config.h +++ b/source/extensions/filters/http/compressor/compressor_library_config.h @@ -1,6 +1,9 @@ #pragma once #include "envoy/config/typed_config.h" +#include "envoy/server/filter_config.h" + +#include "extensions/filters/http/compressor/compressor_factory.h" namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 6db776fdb9a1..9426cc8b3b70 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -11,8 +11,8 @@ namespace Envoy { class CompressorIntegrationTest : public testing::TestWithParam, - public Event::SimulatedTimeSystem, - public HttpIntegrationTest { + public Event::SimulatedTimeSystem, + public HttpIntegrationTest { public: CompressorIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} diff --git a/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc b/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc index 5a2d904917a6..1dff09e8b5c4 100644 --- a/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc +++ b/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc @@ -4,8 +4,6 @@ #include "gtest/gtest.h" -using testing::Return; - namespace Envoy { namespace Extensions { namespace HttpFilters { diff --git a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc index 2285d75f8b00..7a8db6c53dab 100644 --- a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc @@ -1,3 +1,5 @@ +#include + #include "common/common/hex.h" #include "common/decompressor/zlib_decompressor_impl.h" @@ -7,8 +9,6 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/stats/mocks.h" -using testing::Return; - namespace Envoy { namespace Extensions { namespace HttpFilters { @@ -20,8 +20,8 @@ class GzipFilterTest : public testing::Test { envoy::extensions::filters::http::compressor::v3::Compressor compressor; envoy::extensions::filters::http::compressor::gzip::v3::Gzip gzip; CompressorFactoryPtr gzip_factory = std::make_unique(gzip); - config_.reset( - new CompressorFilterConfig(compressor, "test.", stats_, runtime_, std::move(gzip_factory))); + config_ = std::make_shared(compressor, "test.", stats_, runtime_, + std::move(gzip_factory)); filter_ = std::make_unique(config_); filter_->setEncoderFilterCallbacks(encoder_callbacks_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); From fd7fab2f0cf3fec1a839c38b95d04d2989d6dac0 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 30 Mar 2020 10:36:28 +0300 Subject: [PATCH 04/56] coverage: fix naming conflict Signed-off-by: Dmitry Rozhkov --- .../filters/http/compressor/gzip/gzip_filter_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc index 7a8db6c53dab..28cb3b9a21f8 100644 --- a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc @@ -14,7 +14,7 @@ namespace Extensions { namespace HttpFilters { namespace Compressor { -class GzipFilterTest : public testing::Test { +class GzipCompressorFilterTest : public testing::Test { protected: void SetUp() override { envoy::extensions::filters::http::compressor::v3::Compressor compressor; @@ -28,7 +28,7 @@ class GzipFilterTest : public testing::Test { decompressor_.init(31); } - // GzipFilterTest Helpers + // GzipCompressorFilterTest Helpers void verifyCompressedData(const uint32_t content_length) { // This makes sure we have a finished buffer before sending it to the client. expectValidFinishedBuffer(content_length); @@ -131,7 +131,7 @@ class GzipFilterTest : public testing::Test { }; // Acceptance Testing with default configuration. -TEST_F(GzipFilterTest, AcceptanceGzipEncoding) { +TEST_F(GzipCompressorFilterTest, AcceptanceGzipEncoding) { doRequest({{":method", "get"}, {"accept-encoding", "deflate, gzip"}}, false); Http::MetadataMap metadata_map{{"metadata", "metadata"}}; EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); @@ -142,7 +142,7 @@ TEST_F(GzipFilterTest, AcceptanceGzipEncoding) { doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); } -TEST_F(GzipFilterTest, AcceptanceGzipEncodingWithTrailers) { +TEST_F(GzipCompressorFilterTest, AcceptanceGzipEncodingWithTrailers) { doRequest({{":method", "get"}, {"accept-encoding", "deflate, gzip"}}, false); Buffer::OwnedImpl data("hello"); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); @@ -152,13 +152,13 @@ TEST_F(GzipFilterTest, AcceptanceGzipEncodingWithTrailers) { } // Verifies that compression is skipped when accept-encoding header is not allowed. -TEST_F(GzipFilterTest, AcceptEncodingNoCompression) { +TEST_F(GzipCompressorFilterTest, AcceptEncodingNoCompression) { doRequest({{":method", "get"}, {"accept-encoding", "gzip;q=0, deflate"}}, true); doResponseNoCompression({{":method", "get"}, {"content-length", "256"}}); } // Verifies that compression is NOT skipped when accept-encoding header is allowed. -TEST_F(GzipFilterTest, AcceptEncodingCompression) { +TEST_F(GzipCompressorFilterTest, AcceptEncodingCompression) { doRequest({{":method", "get"}, {"accept-encoding", "gzip, deflate"}}, true); doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); } From 7bfd623cc91ebf27fbfd701229883954da727eff Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 30 Mar 2020 14:24:12 +0300 Subject: [PATCH 05/56] fix bazel compile_time_options build Signed-off-by: Dmitry Rozhkov --- .../http/compressor/compressor_filter_integration_test.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 9426cc8b3b70..6eb4b4a34271 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -67,7 +67,8 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Tue, 31 Mar 2020 15:21:08 +0300 Subject: [PATCH 06/56] explicitly deprecate gzip filter Signed-off-by: Dmitry Rozhkov --- .../http/http_filters/compressor_filter.rst | 4 +++ .../http/http_filters/gzip_filter.rst | 5 +++ docs/root/intro/deprecated.rst | 2 ++ source/extensions/filters/http/gzip/config.cc | 19 ++++++++++ test/extensions/filters/http/gzip/BUILD | 2 ++ .../filters/http/gzip/gzip_filter_test.cc | 36 +++++++++++++++++++ 6 files changed, 68 insertions(+) diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst index 2a129f613d0b..4dd9c798488a 100644 --- a/docs/root/configuration/http/http_filters/compressor_filter.rst +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -7,6 +7,10 @@ from an upstream service upon client request. Compression is useful in situations where large payloads need to be transmitted without compromising the response time. +.. note:: + + This filter deprecates the :ref:`HTTP Gzip filter `. + Configuration ------------- * :ref:`v2 API reference ` diff --git a/docs/root/configuration/http/http_filters/gzip_filter.rst b/docs/root/configuration/http/http_filters/gzip_filter.rst index 0251012244f1..70c0c1590849 100644 --- a/docs/root/configuration/http/http_filters/gzip_filter.rst +++ b/docs/root/configuration/http/http_filters/gzip_filter.rst @@ -1,5 +1,10 @@ .. _config_http_filters_gzip: +.. warning:: + + This filter has been deprecated in favor the + :ref:`HTTP Compressor filter `. + Gzip ==== Gzip is an HTTP filter which enables Envoy to compress dispatched data diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index 9c268e81bfd2..959b7b19ed3c 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -92,6 +92,8 @@ Deprecated items below are listed in chronological order. been deprecated in favor of `compressor`. * The statistics counter `header_gzip` in :ref:`HTTP Gzip filter ` has been deprecated in favor of `header_compressor_used`. +* The :ref:`HTTP Gzip filter ` has been deprecated in favor of + :ref:`Compressor `. * Support for the undocumented HTTP/1.1 `:no-chunks` pseudo-header has been removed. If an extension was using this it can achieve the same behavior via the new `http1StreamEncoderOptions()` API. * The grpc_stats filter behavior of by default creating a new stat for every message type seen is deprecated. diff --git a/source/extensions/filters/http/gzip/config.cc b/source/extensions/filters/http/gzip/config.cc index f8c577d67f34..d11d9279d5be 100644 --- a/source/extensions/filters/http/gzip/config.cc +++ b/source/extensions/filters/http/gzip/config.cc @@ -10,6 +10,25 @@ namespace Gzip { Http::FilterFactoryCb GzipFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::gzip::v3::Gzip& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + // This will flip to false eventually. + const bool runtime_feature_default = true; + const char runtime_key[] = "envoy.deprecated_features.allow_deprecated_gzip_http_filter"; + const std::string warn_message = + "Using deprecated extension 'envoy.extensions.filters.http.gzip'. This " + "extension will be removed from Envoy soon. Please use " + "'envoy.extensions.filters.http.compressor' instead."; + + if (context.runtime().snapshot().deprecatedFeatureEnabled(runtime_key, runtime_feature_default)) { + ENVOY_LOG_MISC(warn, "{}", warn_message); + } else { + throw EnvoyException( + warn_message + + " If continued use of this extension is absolutely necessary, see " + "https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime" + "#using-runtime-overrides-for-deprecated-features for how to apply a temporary and " + "highly discouraged override."); + } + Common::Compressors::CompressorFilterConfigSharedPtr config = std::make_shared( proto_config, stats_prefix, context.scope(), context.runtime()); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { diff --git a/test/extensions/filters/http/gzip/BUILD b/test/extensions/filters/http/gzip/BUILD index 72941b500b9f..197789425917 100644 --- a/test/extensions/filters/http/gzip/BUILD +++ b/test/extensions/filters/http/gzip/BUILD @@ -23,6 +23,8 @@ envoy_extension_cc_test( "//source/extensions/filters/http/gzip:gzip_filter_lib", "//test/mocks/http:http_mocks", "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:logging_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/filters/http/gzip/v3:pkg_cc_proto", ], diff --git a/test/extensions/filters/http/gzip/gzip_filter_test.cc b/test/extensions/filters/http/gzip/gzip_filter_test.cc index 01470e0c0ed4..a764274224e1 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_test.cc @@ -7,11 +7,14 @@ #include "common/decompressor/zlib_decompressor_impl.h" #include "common/protobuf/utility.h" +#include "extensions/filters/http/gzip/config.h" #include "extensions/filters/http/gzip/gzip_filter.h" #include "test/mocks/http/mocks.h" #include "test/mocks/runtime/mocks.h" +#include "test/mocks/server/mocks.h" #include "test/mocks/stats/mocks.h" +#include "test/test_common/logging.h" #include "test/test_common/utility.h" #include "absl/container/fixed_array.h" @@ -417,6 +420,39 @@ TEST(GzipFilterConfigTest, DEPRECATED_FEATURE_TEST(DeprecatedExtensionFilterName deprecated_name)); } +// Test that the deprecated extension triggers an exception. +TEST(GzipFilterFactoryTest, DEPRECATED_FEATURE_TEST(TestCheckDeprecatedExtensionThrows)) { + NiceMock context; + GzipFilterFactory factory; + envoy::extensions::filters::http::gzip::v3::Gzip config; + + EXPECT_CALL( + context.runtime_loader_.snapshot_, + deprecatedFeatureEnabled("envoy.deprecated_features.allow_deprecated_gzip_http_filter", _)) + .WillRepeatedly(Return(false)); + + EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(config, "stats.", context), + EnvoyException, + "Using deprecated extension 'envoy.extensions.filters.http.gzip'.*"); +} + +// Test that the deprecated extension gives a deprecation warning. +TEST(GzipFilterFactoryTest, DEPRECATED_FEATURE_TEST(TestCheckDeprecatedExtensionWarns)) { + NiceMock context; + GzipFilterFactory factory; + envoy::extensions::filters::http::gzip::v3::Gzip config; + + EXPECT_CALL( + context.runtime_loader_.snapshot_, + deprecatedFeatureEnabled("envoy.deprecated_features.allow_deprecated_gzip_http_filter", _)) + .WillRepeatedly(Return(true)); + + EXPECT_NO_THROW(factory.createFilterFactoryFromProto(config, "stats.", context)); + + EXPECT_LOG_CONTAINS("warn", "Using deprecated extension 'envoy.extensions.filters.http.gzip'.", + factory.createFilterFactoryFromProto(config, "stats.", context)); +} + } // namespace Gzip } // namespace HttpFilters } // namespace Extensions From d9b33ccdb1258a22b4fa63b6753ca463980e417b Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 1 Apr 2020 12:00:23 +0300 Subject: [PATCH 07/56] fix bazel compile_time_options build Signed-off-by: Dmitry Rozhkov --- .../http/gzip/gzip_filter_integration_test.cc | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc index 83ecbb2c22ad..ad8c658e0502 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc @@ -110,7 +110,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, GzipIntegrationTest, /** * Exercises gzip compression with default configuration. */ -TEST_P(GzipIntegrationTest, AcceptanceDefaultConfigTest) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(AcceptanceDefaultConfigTest)) { initializeFilter(default_config); doRequestAndCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, {":path", "/test/long/url"}, @@ -140,7 +140,7 @@ TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(AcceptanceDeprecatedFullConf /** * Exercises gzip compression with full configuration. */ -TEST_P(GzipIntegrationTest, AcceptanceFullConfigTest) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(AcceptanceFullConfigTest)) { initializeFilter(full_config); doRequestAndCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, {":path", "/test/long/url"}, @@ -155,7 +155,7 @@ TEST_P(GzipIntegrationTest, AcceptanceFullConfigTest) { /** * Exercises filter when client request contains 'identity' type. */ -TEST_P(GzipIntegrationTest, IdentityAcceptEncoding) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(IdentityAcceptEncoding)) { initializeFilter(default_config); doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, {":path", "/test/long/url"}, @@ -170,7 +170,7 @@ TEST_P(GzipIntegrationTest, IdentityAcceptEncoding) { /** * Exercises filter when client request contains unsupported 'accept-encoding' type. */ -TEST_P(GzipIntegrationTest, NotSupportedAcceptEncoding) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(NotSupportedAcceptEncoding)) { initializeFilter(default_config); doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, {":path", "/test/long/url"}, @@ -185,7 +185,7 @@ TEST_P(GzipIntegrationTest, NotSupportedAcceptEncoding) { /** * Exercises filter when upstream response is already encoded. */ -TEST_P(GzipIntegrationTest, UpstreamResponseAlreadyEncoded) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(UpstreamResponseAlreadyEncoded)) { initializeFilter(default_config); Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/test/long/url"}, @@ -211,7 +211,7 @@ TEST_P(GzipIntegrationTest, UpstreamResponseAlreadyEncoded) { /** * Exercises filter when upstream responds with content length below the default threshold. */ -TEST_P(GzipIntegrationTest, NotEnoughContentLength) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(NotEnoughContentLength)) { initializeFilter(default_config); Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/test/long/url"}, @@ -235,7 +235,7 @@ TEST_P(GzipIntegrationTest, NotEnoughContentLength) { /** * Exercises filter when response from upstream service is empty. */ -TEST_P(GzipIntegrationTest, EmptyResponse) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(EmptyResponse)) { initializeFilter(default_config); Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/test/long/url"}, @@ -258,7 +258,7 @@ TEST_P(GzipIntegrationTest, EmptyResponse) { /** * Exercises filter when upstream responds with restricted content-type value. */ -TEST_P(GzipIntegrationTest, SkipOnContentType) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(SkipOnContentType)) { initializeFilter(full_config); doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, {":path", "/test/long/url"}, @@ -273,7 +273,7 @@ TEST_P(GzipIntegrationTest, SkipOnContentType) { /** * Exercises filter when upstream responds with restricted cache-control value. */ -TEST_P(GzipIntegrationTest, SkipOnCacheControl) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(SkipOnCacheControl)) { initializeFilter(full_config); doRequestAndNoCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, {":path", "/test/long/url"}, @@ -289,7 +289,7 @@ TEST_P(GzipIntegrationTest, SkipOnCacheControl) { /** * Exercises gzip compression when upstream returns a chunked response. */ -TEST_P(GzipIntegrationTest, AcceptanceFullConfigChunkedResponse) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(AcceptanceFullConfigChunkedResponse)) { initializeFilter(full_config); Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/test/long/url"}, @@ -313,7 +313,7 @@ TEST_P(GzipIntegrationTest, AcceptanceFullConfigChunkedResponse) { /** * Verify Vary header values are preserved. */ -TEST_P(GzipIntegrationTest, AcceptanceFullConfigVeryHeader) { +TEST_P(GzipIntegrationTest, DEPRECATED_FEATURE_TEST(AcceptanceFullConfigVeryHeader)) { initializeFilter(default_config); Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/test/long/url"}, From 90d02d5f6cae7db5e4b4feac2ebe292f2d3313fe Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 6 Apr 2020 16:16:29 +0300 Subject: [PATCH 08/56] address review comments Signed-off-by: Dmitry Rozhkov --- .../filter/http/compressor/gzip/v2/gzip.proto | 5 ++++ .../http/compressor/v2/compressor.proto | 11 +++----- .../http/compressor/gzip/v3/gzip.proto | 5 ++++ .../http/compressor/v3/compressor.proto | 13 +++------- .../http/http_filters/compressor_filter.rst | 26 +++++++++++++++++-- .../filter/http/compressor/gzip/v2/gzip.proto | 5 ++++ .../http/compressor/v2/compressor.proto | 11 +++----- .../http/compressor/gzip/v3/gzip.proto | 5 ++++ .../http/compressor/v3/compressor.proto | 11 +++----- .../filters/http/compressor/config.cc | 20 +++++--------- .../compressor_filter_integration_test.cc | 10 ++++++- 11 files changed, 72 insertions(+), 50 deletions(-) diff --git a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto index ab6092e24aa4..54bb33b1e265 100644 --- a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -21,9 +21,14 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#next-free-field: 10] message Gzip { enum CompressionStrategy { + // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't + // equate to any other strategy. DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; } diff --git a/api/envoy/config/filter/http/compressor/v2/compressor.proto b/api/envoy/config/filter/http/compressor/v2/compressor.proto index fbe220559bd5..b54ae68632b2 100644 --- a/api/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/api/envoy/config/filter/http/compressor/v2/compressor.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.compressor.v2; import "envoy/api/v2/core/base.proto"; import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/migrate.proto"; @@ -51,9 +50,9 @@ message Compressor { // filter will operate as a pass-through filter. If not specified, defaults to enabled. api.v2.core.RuntimeFeatureFlag runtime_enabled = 5; - // A compressor library to use for compression. By default + // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` - // is used. + // is included in Envoy. // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. @@ -66,9 +65,5 @@ message CompressorLibrary { // Compressor library specific configuration. See the supported libraries for further // documentation. - oneof config_type { - google.protobuf.Struct config = 2 [deprecated = true]; - - google.protobuf.Any typed_config = 3; - } + google.protobuf.Any typed_config = 2; } diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index 855c529f117a..bd57c86cd686 100644 --- a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -22,9 +22,14 @@ message Gzip { "envoy.config.filter.http.compressor.gzip.v2.Gzip"; enum CompressionStrategy { + // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't + // equate to any other strategy. DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; } diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 591193c896b0..a5835059b067 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -5,7 +5,6 @@ package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; @@ -52,9 +51,9 @@ message Compressor { // filter will operate as a pass-through filter. If not specified, defaults to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; - // A compressor library to use for compression. By default + // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` - // is used. + // is included in Envoy. // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. @@ -65,16 +64,10 @@ message CompressorLibrary { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.CompressorLibrary"; - reserved 2; - - reserved "config"; - // The name of the compressor library to instantiate for the filter. string name = 1 [(validate.rules).string = {min_bytes: 1}]; // Compressor library specific configuration. See the supported libraries for further // documentation. - oneof config_type { - google.protobuf.Any typed_config = 3; - } + google.protobuf.Any typed_config = 2; } diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst index 4dd9c798488a..d4eeee2dc140 100644 --- a/docs/root/configuration/http/http_filters/compressor_filter.rst +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -4,8 +4,8 @@ Compressor ========== Compressor is an HTTP filter which enables Envoy to compress dispatched data from an upstream service upon client request. Compression is useful in -situations where large payloads need to be transmitted without -compromising the response time. +situations when bandwidth is scarce and large payloads can be effectively compressed +at the expense of higher CPU load or offloading it to a compression accelerator. .. note:: @@ -26,6 +26,28 @@ response and request allow. Currently the filter supports :ref:`gzip compression` only. Other compression libraries can be supported as extensions. +An example configuration of the filter may look like the following: + +.. code-block:: yaml + + http_filters: + - name: compressor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor + disable_on_etag_header: true + content_length: 100 + content_type: + - text/html + - application/json + compressor_library: + name: gzip + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip + memory_level: 3 + window_bits: 10 + compression_level: best + compression_strategy: rle + By *default* compression will be *skipped* when: - A request does NOT contain *accept-encoding* header. diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto index ab6092e24aa4..54bb33b1e265 100644 --- a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -21,9 +21,14 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#next-free-field: 10] message Gzip { enum CompressionStrategy { + // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't + // equate to any other strategy. DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; } diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto index fbe220559bd5..b54ae68632b2 100644 --- a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.compressor.v2; import "envoy/api/v2/core/base.proto"; import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/migrate.proto"; @@ -51,9 +50,9 @@ message Compressor { // filter will operate as a pass-through filter. If not specified, defaults to enabled. api.v2.core.RuntimeFeatureFlag runtime_enabled = 5; - // A compressor library to use for compression. By default + // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` - // is used. + // is included in Envoy. // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. @@ -66,9 +65,5 @@ message CompressorLibrary { // Compressor library specific configuration. See the supported libraries for further // documentation. - oneof config_type { - google.protobuf.Struct config = 2 [deprecated = true]; - - google.protobuf.Any typed_config = 3; - } + google.protobuf.Any typed_config = 2; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index 855c529f117a..bd57c86cd686 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -22,9 +22,14 @@ message Gzip { "envoy.config.filter.http.compressor.gzip.v2.Gzip"; enum CompressionStrategy { + // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't + // equate to any other strategy. DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index fc123541e692..a5835059b067 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -5,7 +5,6 @@ package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; @@ -52,9 +51,9 @@ message Compressor { // filter will operate as a pass-through filter. If not specified, defaults to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; - // A compressor library to use for compression. By default + // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` - // is used. + // is included in Envoy. // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. @@ -70,9 +69,5 @@ message CompressorLibrary { // Compressor library specific configuration. See the supported libraries for further // documentation. - oneof config_type { - google.protobuf.Struct hidden_envoy_deprecated_config = 2 [deprecated = true]; - - google.protobuf.Any typed_config = 3; - } + google.protobuf.Any typed_config = 2; } diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc index 5d22f8a91171..5a2266139e5a 100644 --- a/source/extensions/filters/http/compressor/config.cc +++ b/source/extensions/filters/http/compressor/config.cc @@ -14,19 +14,13 @@ namespace Compressor { Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::compressor::v3::Compressor& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { - CompressorFactoryPtr compressor_factory; - - if (proto_config.has_compressor_library()) { - auto& config_factory = Config::Utility::getAndCheckFactory( - proto_config.compressor_library()); - ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( - proto_config.compressor_library(), context.messageValidationVisitor(), config_factory); - compressor_factory = config_factory.createCompressorLibraryFromProto(*message, context); - } else { - // By default use gzip compression. - compressor_factory = std::make_unique( - envoy::extensions::filters::http::compressor::gzip::v3::Gzip()); - } + auto& config_factory = Config::Utility::getAndCheckFactory( + proto_config.compressor_library()); + ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig( + proto_config.compressor_library().typed_config(), context.messageValidationVisitor(), + config_factory); + CompressorFactoryPtr compressor_factory = + config_factory.createCompressorLibraryFromProto(*message, context); Common::Compressors::CompressorFilterConfigSharedPtr config = std::make_shared(proto_config, stats_prefix, context.scope(), context.runtime(), std::move(compressor_factory)); diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 6eb4b4a34271..f5fd2ba793d1 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -103,7 +103,15 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Tue, 7 Apr 2020 12:47:58 +0300 Subject: [PATCH 09/56] zlib: use Z_DEFAULT_STRATEGY by default instead of Z_FIXED Since Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy is simply static_cast'ed to uint64_t the Standard strategy (4) becomes Z_FIXED (4 as well). This basically disables the use of dynamic Huffman codes when the gzip filter is configured to use default values. Make the Standard strategy equal to 0 to translate to Z_DEFAULT_STRATEGY. Contributes to #8448 Signed-off-by: Dmitry Rozhkov --- source/common/compressor/zlib_compressor_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index ecfdc8aa4163..5889cb1497e9 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -48,7 +48,7 @@ class ZlibCompressorImpl : public Compressor { Filtered = 1, Huffman = 2, Rle = 3, - Standard = 4, + Standard = 0, }; /** From c3c26b670ef46238a2de81ea6002287262b78048 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 7 Apr 2020 11:56:36 +0300 Subject: [PATCH 10/56] gzip.proto: rework CompressionStrategy and CompressionLevel enums Signed-off-by: Dmitry Rozhkov --- .../filter/http/compressor/gzip/v2/gzip.proto | 60 ++++++++++-------- .../http/compressor/gzip/v3/gzip.proto | 63 ++++++++++--------- .../filter/http/compressor/gzip/v2/gzip.proto | 60 ++++++++++-------- .../http/compressor/gzip/v3/gzip.proto | 63 ++++++++++--------- .../common/compressor/zlib_compressor_impl.h | 10 +++ .../filters/http/compressor/gzip/config.cc | 30 ++++++--- .../filters/http/compressor/gzip/config.h | 2 +- .../compressor_filter_integration_test.cc | 4 +- .../gzip/gzip_compressor_library_test.cc | 39 ++++++++++-- 9 files changed, 205 insertions(+), 126 deletions(-) diff --git a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto index 54bb33b1e265..fcb3d65f675a 100644 --- a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -18,26 +18,32 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] -// [#next-free-field: 10] message Gzip { + // All the values of this enumeration translate directly to zlib's compression strategies. + // For more information about each strategy, please refer to zlib manual. enum CompressionStrategy { - // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't - // equate to any other strategy. - DEFAULT = 0; - + DEFAULT_STRATEGY = 0; FILTERED = 1; - - HUFFMAN = 2; - + HUFFMAN_ONLY = 2; RLE = 3; + FIXED = 4; } - message CompressionLevel { - enum Enum { - DEFAULT = 0; - BEST = 1; - SPEED = 2; - } + enum CompressionLevel { + option allow_alias = true; + + DEFAULT_COMPRESSION = 0; + BEST_SPEED = 1; + COMPRESSION_LEVEL_1 = 1; + COMPRESSION_LEVEL_2 = 2; + COMPRESSION_LEVEL_3 = 3; + COMPRESSION_LEVEL_4 = 4; + COMPRESSION_LEVEL_5 = 5; + COMPRESSION_LEVEL_6 = 6; + COMPRESSION_LEVEL_7 = 7; + COMPRESSION_LEVEL_8 = 8; + COMPRESSION_LEVEL_9 = 9; + BEST_COMPRESSION = 9; } // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values @@ -45,23 +51,25 @@ message Gzip { google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST" provides higher compression at the cost of - // higher latency, "SPEED" provides lower compression with minimum impact on response time. - // "DEFAULT" provides an optimal result between speed and compression. This field will be set to - // "DEFAULT" if not specified. - CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + // amount of compression applied to the content. "BEST_COMPRESSION" provides higher compression + // at the cost of higher latency and is equal to "COMPRESSION_LEVEL_9". "BEST_SPEED" provides + // lower compression with minimum impact on response time, the same as "COMPRESSION_LEVEL_1". + // "DEFAULT_COMPRESSION" provides an optimal result between speed and compression. According + // to zlib's manual this level gives the same result as "COMPRESSION_LEVEL_6". + // This field will be set to "DEFAULT_COMPRESSION" if not specified. + CompressionLevel compression_level = 2 [(validate.rules).enum = {defined_only: true}]; // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though - // there are situations which changing this parameter might produce better results. For example, - // run-length encoding (RLE) is typically used when the content is known for having sequences - // which same data occurs many consecutive times. For more information about each strategy, please - // refer to zlib manual. - CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + // characteristics of the content. Most of the time "DEFAULT_STRATEGY" will be the best choice, + // which is also the default value for the parameter, though there are situations when + // changing this parameter might produce better results. For example, run-length encoding (RLE) + // is typically used when the content is known for having sequences which same data occurs many + // consecutive times. For more information about each strategy, please refer to zlib manual. + CompressionStrategy compression_strategy = 3 [(validate.rules).enum = {defined_only: true}]; // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. // Larger window results in better compression at the expense of memory usage. The default is 12 // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; } diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index bd57c86cd686..b9d24e145ca2 100644 --- a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -16,32 +16,35 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] -// [#next-free-field: 10] message Gzip { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.gzip.v2.Gzip"; + // All the values of this enumeration translate directly to zlib's compression strategies. + // For more information about each strategy, please refer to zlib manual. enum CompressionStrategy { - // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't - // equate to any other strategy. - DEFAULT = 0; - + DEFAULT_STRATEGY = 0; FILTERED = 1; - - HUFFMAN = 2; - + HUFFMAN_ONLY = 2; RLE = 3; + FIXED = 4; } - message CompressionLevel { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.compressor.gzip.v2.Gzip.CompressionLevel"; - - enum Enum { - DEFAULT = 0; - BEST = 1; - SPEED = 2; - } + enum CompressionLevel { + option allow_alias = true; + + DEFAULT_COMPRESSION = 0; + BEST_SPEED = 1; + COMPRESSION_LEVEL_1 = 1; + COMPRESSION_LEVEL_2 = 2; + COMPRESSION_LEVEL_3 = 3; + COMPRESSION_LEVEL_4 = 4; + COMPRESSION_LEVEL_5 = 5; + COMPRESSION_LEVEL_6 = 6; + COMPRESSION_LEVEL_7 = 7; + COMPRESSION_LEVEL_8 = 8; + COMPRESSION_LEVEL_9 = 9; + BEST_COMPRESSION = 9; } // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values @@ -49,23 +52,25 @@ message Gzip { google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST" provides higher compression at the cost of - // higher latency, "SPEED" provides lower compression with minimum impact on response time. - // "DEFAULT" provides an optimal result between speed and compression. This field will be set to - // "DEFAULT" if not specified. - CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + // amount of compression applied to the content. "BEST_COMPRESSION" provides higher compression + // at the cost of higher latency and is equal to "COMPRESSION_LEVEL_9". "BEST_SPEED" provides + // lower compression with minimum impact on response time, the same as "COMPRESSION_LEVEL_1". + // "DEFAULT_COMPRESSION" provides an optimal result between speed and compression. According + // to zlib's manual this level gives the same result as "COMPRESSION_LEVEL_6". + // This field will be set to "DEFAULT_COMPRESSION" if not specified. + CompressionLevel compression_level = 2 [(validate.rules).enum = {defined_only: true}]; // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though - // there are situations which changing this parameter might produce better results. For example, - // run-length encoding (RLE) is typically used when the content is known for having sequences - // which same data occurs many consecutive times. For more information about each strategy, please - // refer to zlib manual. - CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + // characteristics of the content. Most of the time "DEFAULT_STRATEGY" will be the best choice, + // which is also the default value for the parameter, though there are situations when + // changing this parameter might produce better results. For example, run-length encoding (RLE) + // is typically used when the content is known for having sequences which same data occurs many + // consecutive times. For more information about each strategy, please refer to zlib manual. + CompressionStrategy compression_strategy = 3 [(validate.rules).enum = {defined_only: true}]; // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. // Larger window results in better compression at the expense of memory usage. The default is 12 // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; } diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto index 54bb33b1e265..fcb3d65f675a 100644 --- a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -18,26 +18,32 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] -// [#next-free-field: 10] message Gzip { + // All the values of this enumeration translate directly to zlib's compression strategies. + // For more information about each strategy, please refer to zlib manual. enum CompressionStrategy { - // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't - // equate to any other strategy. - DEFAULT = 0; - + DEFAULT_STRATEGY = 0; FILTERED = 1; - - HUFFMAN = 2; - + HUFFMAN_ONLY = 2; RLE = 3; + FIXED = 4; } - message CompressionLevel { - enum Enum { - DEFAULT = 0; - BEST = 1; - SPEED = 2; - } + enum CompressionLevel { + option allow_alias = true; + + DEFAULT_COMPRESSION = 0; + BEST_SPEED = 1; + COMPRESSION_LEVEL_1 = 1; + COMPRESSION_LEVEL_2 = 2; + COMPRESSION_LEVEL_3 = 3; + COMPRESSION_LEVEL_4 = 4; + COMPRESSION_LEVEL_5 = 5; + COMPRESSION_LEVEL_6 = 6; + COMPRESSION_LEVEL_7 = 7; + COMPRESSION_LEVEL_8 = 8; + COMPRESSION_LEVEL_9 = 9; + BEST_COMPRESSION = 9; } // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values @@ -45,23 +51,25 @@ message Gzip { google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST" provides higher compression at the cost of - // higher latency, "SPEED" provides lower compression with minimum impact on response time. - // "DEFAULT" provides an optimal result between speed and compression. This field will be set to - // "DEFAULT" if not specified. - CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + // amount of compression applied to the content. "BEST_COMPRESSION" provides higher compression + // at the cost of higher latency and is equal to "COMPRESSION_LEVEL_9". "BEST_SPEED" provides + // lower compression with minimum impact on response time, the same as "COMPRESSION_LEVEL_1". + // "DEFAULT_COMPRESSION" provides an optimal result between speed and compression. According + // to zlib's manual this level gives the same result as "COMPRESSION_LEVEL_6". + // This field will be set to "DEFAULT_COMPRESSION" if not specified. + CompressionLevel compression_level = 2 [(validate.rules).enum = {defined_only: true}]; // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though - // there are situations which changing this parameter might produce better results. For example, - // run-length encoding (RLE) is typically used when the content is known for having sequences - // which same data occurs many consecutive times. For more information about each strategy, please - // refer to zlib manual. - CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + // characteristics of the content. Most of the time "DEFAULT_STRATEGY" will be the best choice, + // which is also the default value for the parameter, though there are situations when + // changing this parameter might produce better results. For example, run-length encoding (RLE) + // is typically used when the content is known for having sequences which same data occurs many + // consecutive times. For more information about each strategy, please refer to zlib manual. + CompressionStrategy compression_strategy = 3 [(validate.rules).enum = {defined_only: true}]; // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. // Larger window results in better compression at the expense of memory usage. The default is 12 // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index bd57c86cd686..b9d24e145ca2 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -16,32 +16,35 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] -// [#next-free-field: 10] message Gzip { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.gzip.v2.Gzip"; + // All the values of this enumeration translate directly to zlib's compression strategies. + // For more information about each strategy, please refer to zlib manual. enum CompressionStrategy { - // The DEFAULT value translates directly to zlib's Z_DEFAULT_STRATEGY. It doesn't - // equate to any other strategy. - DEFAULT = 0; - + DEFAULT_STRATEGY = 0; FILTERED = 1; - - HUFFMAN = 2; - + HUFFMAN_ONLY = 2; RLE = 3; + FIXED = 4; } - message CompressionLevel { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.compressor.gzip.v2.Gzip.CompressionLevel"; - - enum Enum { - DEFAULT = 0; - BEST = 1; - SPEED = 2; - } + enum CompressionLevel { + option allow_alias = true; + + DEFAULT_COMPRESSION = 0; + BEST_SPEED = 1; + COMPRESSION_LEVEL_1 = 1; + COMPRESSION_LEVEL_2 = 2; + COMPRESSION_LEVEL_3 = 3; + COMPRESSION_LEVEL_4 = 4; + COMPRESSION_LEVEL_5 = 5; + COMPRESSION_LEVEL_6 = 6; + COMPRESSION_LEVEL_7 = 7; + COMPRESSION_LEVEL_8 = 8; + COMPRESSION_LEVEL_9 = 9; + BEST_COMPRESSION = 9; } // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values @@ -49,23 +52,25 @@ message Gzip { google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST" provides higher compression at the cost of - // higher latency, "SPEED" provides lower compression with minimum impact on response time. - // "DEFAULT" provides an optimal result between speed and compression. This field will be set to - // "DEFAULT" if not specified. - CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + // amount of compression applied to the content. "BEST_COMPRESSION" provides higher compression + // at the cost of higher latency and is equal to "COMPRESSION_LEVEL_9". "BEST_SPEED" provides + // lower compression with minimum impact on response time, the same as "COMPRESSION_LEVEL_1". + // "DEFAULT_COMPRESSION" provides an optimal result between speed and compression. According + // to zlib's manual this level gives the same result as "COMPRESSION_LEVEL_6". + // This field will be set to "DEFAULT_COMPRESSION" if not specified. + CompressionLevel compression_level = 2 [(validate.rules).enum = {defined_only: true}]; // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though - // there are situations which changing this parameter might produce better results. For example, - // run-length encoding (RLE) is typically used when the content is known for having sequences - // which same data occurs many consecutive times. For more information about each strategy, please - // refer to zlib manual. - CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + // characteristics of the content. Most of the time "DEFAULT_STRATEGY" will be the best choice, + // which is also the default value for the parameter, though there are situations when + // changing this parameter might produce better results. For example, run-length encoding (RLE) + // is typically used when the content is known for having sequences which same data occurs many + // consecutive times. For more information about each strategy, please refer to zlib manual. + CompressionStrategy compression_strategy = 3 [(validate.rules).enum = {defined_only: true}]; // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. // Larger window results in better compression at the expense of memory usage. The default is 12 // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; } diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 5889cb1497e9..6fe305d2458c 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -33,6 +33,15 @@ class ZlibCompressorImpl : public Compressor { */ enum class CompressionLevel : int64_t { Best = 9, + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9, Speed = 1, Standard = -1, }; @@ -46,6 +55,7 @@ class ZlibCompressorImpl : public Compressor { */ enum class CompressionStrategy : uint64_t { Filtered = 1, + Fixed = 4, Huffman = 2, Rle = 3, Standard = 0, diff --git a/source/extensions/filters/http/compressor/gzip/config.cc b/source/extensions/filters/http/compressor/gzip/config.cc index c47d33f13390..991228412d23 100644 --- a/source/extensions/filters/http/compressor/gzip/config.cc +++ b/source/extensions/filters/http/compressor/gzip/config.cc @@ -26,13 +26,27 @@ GzipCompressorFactory::GzipCompressorFactory( window_bits_(windowBitsUint(gzip.window_bits().value())) {} Envoy::Compressor::ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::compressionLevelEnum( - envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::Enum + envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel compression_level) { switch (compression_level) { - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::BEST: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::SPEED: + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::BEST_SPEED: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Speed; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_2: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level2; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_3: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level3; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_4: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level4; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_5: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level5; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_6: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level6; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_7: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level7; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_8: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level8; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::BEST_COMPRESSION: + return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best; default: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard; } @@ -43,12 +57,14 @@ GzipCompressorFactory::compressionStrategyEnum( envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionStrategy compression_strategy) { switch (compression_strategy) { - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::RLE: - return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle; case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::FILTERED: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::HUFFMAN: + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::FIXED: + return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::HUFFMAN_ONLY: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman; + case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::RLE: + return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle; default: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard; } diff --git a/source/extensions/filters/http/compressor/gzip/config.h b/source/extensions/filters/http/compressor/gzip/config.h index 02a925a8dd9e..321e3431da04 100644 --- a/source/extensions/filters/http/compressor/gzip/config.h +++ b/source/extensions/filters/http/compressor/gzip/config.h @@ -38,7 +38,7 @@ class GzipCompressorFactory : public Compressor::CompressorFactory { friend class GzipTest; static Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compressionLevelEnum( - envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel::Enum + envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel compression_level); static Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategyEnum( envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionStrategy diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index f5fd2ba793d1..f33ddb5ce17f 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -80,7 +80,7 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Tue, 7 Apr 2020 15:05:22 +0300 Subject: [PATCH 11/56] compressor: make compressor_library required Signed-off-by: Dmitry Rozhkov --- .../filter/http/compressor/v2/compressor.proto | 7 ++----- .../filters/http/compressor/v3/compressor.proto | 7 ++----- .../filter/http/compressor/v2/compressor.proto | 7 ++----- .../filters/http/compressor/v3/compressor.proto | 7 ++----- .../extensions/filters/http/compressor/config.cc | 14 ++++++++++---- .../compressor_filter_integration_test.cc | 3 --- 6 files changed, 18 insertions(+), 27 deletions(-) diff --git a/api/envoy/config/filter/http/compressor/v2/compressor.proto b/api/envoy/config/filter/http/compressor/v2/compressor.proto index b54ae68632b2..69aaacafb762 100644 --- a/api/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/api/envoy/config/filter/http/compressor/v2/compressor.proto @@ -56,14 +56,11 @@ message Compressor { // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. - CompressorLibrary compressor_library = 6; + CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } message CompressorLibrary { - // The name of the compressor library to instantiate for the filter. - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - // Compressor library specific configuration. See the supported libraries for further // documentation. - google.protobuf.Any typed_config = 2; + google.protobuf.Any typed_config = 1; } diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index a5835059b067..71161d17f131 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -57,17 +57,14 @@ message Compressor { // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. - CompressorLibrary compressor_library = 6; + CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } message CompressorLibrary { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.CompressorLibrary"; - // The name of the compressor library to instantiate for the filter. - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - // Compressor library specific configuration. See the supported libraries for further // documentation. - google.protobuf.Any typed_config = 2; + google.protobuf.Any typed_config = 1; } diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto index b54ae68632b2..69aaacafb762 100644 --- a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto @@ -56,14 +56,11 @@ message Compressor { // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. - CompressorLibrary compressor_library = 6; + CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } message CompressorLibrary { - // The name of the compressor library to instantiate for the filter. - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - // Compressor library specific configuration. See the supported libraries for further // documentation. - google.protobuf.Any typed_config = 2; + google.protobuf.Any typed_config = 1; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index a5835059b067..71161d17f131 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -57,17 +57,14 @@ message Compressor { // // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" // filter. - CompressorLibrary compressor_library = 6; + CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } message CompressorLibrary { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.CompressorLibrary"; - // The name of the compressor library to instantiate for the filter. - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - // Compressor library specific configuration. See the supported libraries for further // documentation. - google.protobuf.Any typed_config = 2; + google.protobuf.Any typed_config = 1; } diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc index 5a2266139e5a..ade20f1e56d7 100644 --- a/source/extensions/filters/http/compressor/config.cc +++ b/source/extensions/filters/http/compressor/config.cc @@ -14,13 +14,19 @@ namespace Compressor { Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::compressor::v3::Compressor& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { - auto& config_factory = Config::Utility::getAndCheckFactory( - proto_config.compressor_library()); + const std::string type{TypeUtil::typeUrlToDescriptorFullName( + proto_config.compressor_library().typed_config().type_url())}; + NamedCompressorLibraryConfigFactory* const config_factory = + Registry::FactoryRegistry::getFactoryByType(type); + if (config_factory == nullptr) { + throw EnvoyException( + fmt::format("Didn't find a registered implementation for type: '{}'", type)); + } ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig( proto_config.compressor_library().typed_config(), context.messageValidationVisitor(), - config_factory); + *config_factory); CompressorFactoryPtr compressor_factory = - config_factory.createCompressorLibraryFromProto(*message, context); + config_factory->createCompressorLibraryFromProto(*message, context); Common::Compressors::CompressorFilterConfigSharedPtr config = std::make_shared(proto_config, stats_prefix, context.scope(), context.runtime(), std::move(compressor_factory)); diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index f33ddb5ce17f..25af409f28fe 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -75,7 +75,6 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Tue, 7 Apr 2020 16:05:15 +0300 Subject: [PATCH 12/56] gzip: allow setting of chunk size This is a "cherry-pick" of #10508 Signed-off-by: Dmitry Rozhkov --- api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto | 6 ++++++ .../extensions/filters/http/compressor/gzip/v3/gzip.proto | 6 ++++++ .../envoy/config/filter/http/compressor/gzip/v2/gzip.proto | 6 ++++++ .../extensions/filters/http/compressor/gzip/v3/gzip.proto | 6 ++++++ source/extensions/filters/http/compressor/gzip/config.cc | 5 +++-- source/extensions/filters/http/compressor/gzip/config.h | 5 +++-- .../http/compressor/gzip/gzip_compressor_library_test.cc | 5 ++++- 7 files changed, 34 insertions(+), 5 deletions(-) diff --git a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto index fcb3d65f675a..616e55269d7a 100644 --- a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -18,6 +18,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] +// [#next-free-field: 6] message Gzip { // All the values of this enumeration translate directly to zlib's compression strategies. // For more information about each strategy, please refer to zlib manual. @@ -72,4 +73,9 @@ message Gzip { // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + + // Value for Zlib's next output buffer. If not set, defaults to 4096. + // See https://www.zlib.net/manual.html for more details. Also see + // https://github.com/envoyproxy/envoy/issues/8448 for context on this filter's performance. + google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; } diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index b9d24e145ca2..362e1c42c5ad 100644 --- a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -16,6 +16,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] +// [#next-free-field: 6] message Gzip { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.gzip.v2.Gzip"; @@ -73,4 +74,9 @@ message Gzip { // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + + // Value for Zlib's next output buffer. If not set, defaults to 4096. + // See https://www.zlib.net/manual.html for more details. Also see + // https://github.com/envoyproxy/envoy/issues/8448 for context on this filter's performance. + google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; } diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto index fcb3d65f675a..616e55269d7a 100644 --- a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto @@ -18,6 +18,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] +// [#next-free-field: 6] message Gzip { // All the values of this enumeration translate directly to zlib's compression strategies. // For more information about each strategy, please refer to zlib manual. @@ -72,4 +73,9 @@ message Gzip { // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + + // Value for Zlib's next output buffer. If not set, defaults to 4096. + // See https://www.zlib.net/manual.html for more details. Also see + // https://github.com/envoyproxy/envoy/issues/8448 for context on this filter's performance. + google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index b9d24e145ca2..362e1c42c5ad 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -16,6 +16,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] +// [#next-free-field: 6] message Gzip { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.gzip.v2.Gzip"; @@ -73,4 +74,9 @@ message Gzip { // which will produce a 4096 bytes window. For more details about this parameter, please refer to // zlib manual > deflateInit2. google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + + // Value for Zlib's next output buffer. If not set, defaults to 4096. + // See https://www.zlib.net/manual.html for more details. Also see + // https://github.com/envoyproxy/envoy/issues/8448 for context on this filter's performance. + google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; } diff --git a/source/extensions/filters/http/compressor/gzip/config.cc b/source/extensions/filters/http/compressor/gzip/config.cc index 991228412d23..d9db5830702b 100644 --- a/source/extensions/filters/http/compressor/gzip/config.cc +++ b/source/extensions/filters/http/compressor/gzip/config.cc @@ -23,7 +23,8 @@ GzipCompressorFactory::GzipCompressorFactory( : compression_level_(compressionLevelEnum(gzip.compression_level())), compression_strategy_(compressionStrategyEnum(gzip.compression_strategy())), memory_level_(memoryLevelUint(gzip.memory_level().value())), - window_bits_(windowBitsUint(gzip.window_bits().value())) {} + window_bits_(windowBitsUint(gzip.window_bits().value())), + chunk_size_(gzip.chunk_size().value() > 0 ? gzip.chunk_size().value() : 4096) {} Envoy::Compressor::ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::compressionLevelEnum( envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel @@ -79,7 +80,7 @@ uint64_t GzipCompressorFactory::windowBitsUint(Protobuf::uint32 window_bits) { } std::unique_ptr GzipCompressorFactory::createCompressor() { - auto compressor = std::make_unique(); + auto compressor = std::make_unique(chunk_size_); compressor->init(compression_level_, compression_strategy_, window_bits_, memory_level_); return compressor; } diff --git a/source/extensions/filters/http/compressor/gzip/config.h b/source/extensions/filters/http/compressor/gzip/config.h index 321e3431da04..5f93612725b2 100644 --- a/source/extensions/filters/http/compressor/gzip/config.h +++ b/source/extensions/filters/http/compressor/gzip/config.h @@ -50,8 +50,9 @@ class GzipCompressorFactory : public Compressor::CompressorFactory { Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compression_level_; Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compression_strategy_; - int32_t memory_level_; - int32_t window_bits_; + const int32_t memory_level_; + const int32_t window_bits_; + const uint32_t chunk_size_; }; class GzipCompressorLibraryFactory diff --git a/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc b/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc index 39eaffdd8e5d..90b7405f354d 100644 --- a/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc +++ b/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc @@ -31,18 +31,20 @@ class GzipTest : public testing::Test { int32_t memoryLevel() const { return factory_->memory_level_; } int32_t windowBits() const { return factory_->window_bits_; } + int32_t chunkSize() const { return factory_->chunk_size_; } void expectValidCompressionStrategyAndLevel( Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy strategy, absl::string_view strategy_name, Envoy::Compressor::ZlibCompressorImpl::CompressionLevel level, absl::string_view level_name) { setUpGzip(fmt::format( - R"EOF({{"compression_strategy": "{}", "compression_level": "{}", "memory_level": 6, "window_bits": 27}})EOF", + R"EOF({{"compression_strategy": "{}", "compression_level": "{}", "memory_level": 6, "window_bits": 27, "chunk_size": 10000}})EOF", strategy_name, level_name)); EXPECT_EQ(strategy, compressionStrategy()); EXPECT_EQ(level, compressionLevel()); EXPECT_EQ(6, memoryLevel()); EXPECT_EQ(27, windowBits()); + EXPECT_EQ(10000, chunkSize()); } std::unique_ptr factory_; @@ -52,6 +54,7 @@ class GzipTest : public testing::Test { TEST_F(GzipTest, DefaultConfigValues) { EXPECT_EQ(5, memoryLevel()); EXPECT_EQ(28, windowBits()); + EXPECT_EQ(4096, chunkSize()); EXPECT_EQ(Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, compressionStrategy()); EXPECT_EQ(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, compressionLevel()); From b7e1d2836742fc0679d513abc7187dc1f7c10d85 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 8 Apr 2020 13:47:33 +0300 Subject: [PATCH 13/56] document the Fixed compression strategy Signed-off-by: Dmitry Rozhkov --- source/common/compressor/zlib_compressor_impl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 6fe305d2458c..980b02ce32c3 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -49,6 +49,7 @@ class ZlibCompressorImpl : public Compressor { /** * Enum values are used for setting the compression algorithm strategy. * filtered: used for data produced by a filter. (or predictor) @see Z_FILTERED (zlib manual) + * fixed: disable dynamic Huffman codes. @see Z_FIXED (zlib manual) * huffman: used to enforce Huffman encoding. @see RFC 1951 * rle: used to limit match distances to one. (Run-length encoding) * standard: used for normal data. (default) @see Z_DEFAULT_STRATEGY in zlib manual. From 681d7c4b68b38eca5fd8d18028201b5c15e051da Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 8 Apr 2020 15:11:47 +0300 Subject: [PATCH 14/56] move to v3 proto Signed-off-by: Dmitry Rozhkov --- api/BUILD | 1 - .../filter/http/compressor/gzip/v2/BUILD | 9 - .../filter/http/compressor/gzip/v2/gzip.proto | 81 --------- .../http/compressor/v2/compressor.proto | 20 +-- .../filters/http/compressor/gzip/v3/BUILD | 5 +- .../http/compressor/gzip/v3/gzip.proto | 5 +- .../http/compressor/v3/compressor.proto | 3 - .../extensions/filters/http/gzip/v3/BUILD | 2 +- .../filters/http/gzip/v3/gzip.proto | 4 +- .../filters/http/gzip/v4alpha/BUILD | 13 ++ .../filters/http/gzip/v4alpha/gzip.proto | 79 +++++++++ api/versioning/BUILD | 2 +- .../filter/http/compressor/gzip/v2/BUILD | 9 - .../filter/http/compressor/gzip/v2/gzip.proto | 81 --------- .../http/compressor/v2/compressor.proto | 22 +-- .../filters/http/compressor/gzip/v3/BUILD | 5 +- .../http/compressor/gzip/v3/gzip.proto | 5 +- .../http/compressor/v3/compressor.proto | 5 +- .../extensions/filters/http/gzip/v3/BUILD | 2 +- .../filters/http/gzip/v3/gzip.proto | 4 +- .../filters/http/gzip/v4alpha/BUILD | 13 ++ .../filters/http/gzip/v4alpha/gzip.proto | 79 +++++++++ source/extensions/filters/http/gzip/BUILD | 1 + .../filters/http/gzip/gzip_filter.cc | 24 ++- .../filters/http/common/compressor/BUILD | 1 + .../compressor/compressor_filter_test.cc | 162 ++++++++++++++++-- .../compressor_filter_integration_test.cc | 33 ---- .../http/compressor/gzip/gzip_filter_test.cc | 2 +- 28 files changed, 371 insertions(+), 301 deletions(-) delete mode 100644 api/envoy/config/filter/http/compressor/gzip/v2/BUILD delete mode 100644 api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto create mode 100644 api/envoy/extensions/filters/http/gzip/v4alpha/BUILD create mode 100644 api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto delete mode 100644 generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD delete mode 100644 generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto create mode 100644 generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD create mode 100644 generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto diff --git a/api/BUILD b/api/BUILD index 7fd9b969ab9c..9bbbeab31e38 100644 --- a/api/BUILD +++ b/api/BUILD @@ -32,7 +32,6 @@ proto_library( "//envoy/config/filter/http/aws_request_signing/v2alpha:pkg", "//envoy/config/filter/http/buffer/v2:pkg", "//envoy/config/filter/http/cache/v2alpha:pkg", - "//envoy/config/filter/http/compressor/gzip/v2:pkg", "//envoy/config/filter/http/compressor/v2:pkg", "//envoy/config/filter/http/cors/v2:pkg", "//envoy/config/filter/http/csrf/v2:pkg", diff --git a/api/envoy/config/filter/http/compressor/gzip/v2/BUILD b/api/envoy/config/filter/http/compressor/gzip/v2/BUILD deleted file mode 100644 index ef3541ebcb1d..000000000000 --- a/api/envoy/config/filter/http/compressor/gzip/v2/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -# DO NOT EDIT. This file is generated by tools/proto_sync.py. - -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], -) diff --git a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto deleted file mode 100644 index 616e55269d7a..000000000000 --- a/api/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ /dev/null @@ -1,81 +0,0 @@ -syntax = "proto3"; - -package envoy.config.filter.http.compressor.gzip.v2; - -import "google/protobuf/wrappers.proto"; - -import "udpa/annotations/migrate.proto"; -import "udpa/annotations/status.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.gzip.v2"; -option java_outer_classname = "GzipProto"; -option java_multiple_files = true; -option (udpa.annotations.file_migrate).move_to_package = - "envoy.extensions.filters.http.compressor.gzip.v3"; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: Gzip] -// [#extension: envoy.filters.http.compressor.gzip] - -// [#next-free-field: 6] -message Gzip { - // All the values of this enumeration translate directly to zlib's compression strategies. - // For more information about each strategy, please refer to zlib manual. - enum CompressionStrategy { - DEFAULT_STRATEGY = 0; - FILTERED = 1; - HUFFMAN_ONLY = 2; - RLE = 3; - FIXED = 4; - } - - enum CompressionLevel { - option allow_alias = true; - - DEFAULT_COMPRESSION = 0; - BEST_SPEED = 1; - COMPRESSION_LEVEL_1 = 1; - COMPRESSION_LEVEL_2 = 2; - COMPRESSION_LEVEL_3 = 3; - COMPRESSION_LEVEL_4 = 4; - COMPRESSION_LEVEL_5 = 5; - COMPRESSION_LEVEL_6 = 6; - COMPRESSION_LEVEL_7 = 7; - COMPRESSION_LEVEL_8 = 8; - COMPRESSION_LEVEL_9 = 9; - BEST_COMPRESSION = 9; - } - - // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values - // use more memory, but are faster and produce better compression results. The default value is 5. - google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; - - // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST_COMPRESSION" provides higher compression - // at the cost of higher latency and is equal to "COMPRESSION_LEVEL_9". "BEST_SPEED" provides - // lower compression with minimum impact on response time, the same as "COMPRESSION_LEVEL_1". - // "DEFAULT_COMPRESSION" provides an optimal result between speed and compression. According - // to zlib's manual this level gives the same result as "COMPRESSION_LEVEL_6". - // This field will be set to "DEFAULT_COMPRESSION" if not specified. - CompressionLevel compression_level = 2 [(validate.rules).enum = {defined_only: true}]; - - // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT_STRATEGY" will be the best choice, - // which is also the default value for the parameter, though there are situations when - // changing this parameter might produce better results. For example, run-length encoding (RLE) - // is typically used when the content is known for having sequences which same data occurs many - // consecutive times. For more information about each strategy, please refer to zlib manual. - CompressionStrategy compression_strategy = 3 [(validate.rules).enum = {defined_only: true}]; - - // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. - // Larger window results in better compression at the expense of memory usage. The default is 12 - // which will produce a 4096 bytes window. For more details about this parameter, please refer to - // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; - - // Value for Zlib's next output buffer. If not set, defaults to 4096. - // See https://www.zlib.net/manual.html for more details. Also see - // https://github.com/envoyproxy/envoy/issues/8448 for context on this filter's performance. - google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; -} diff --git a/api/envoy/config/filter/http/compressor/v2/compressor.proto b/api/envoy/config/filter/http/compressor/v2/compressor.proto index 922540695909..d62d0d7a42fa 100644 --- a/api/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/api/envoy/config/filter/http/compressor/v2/compressor.proto @@ -4,12 +4,10 @@ package envoy.config.filter.http.compressor.v2; import "envoy/api/v2/core/base.proto"; -import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; -import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.v2"; option java_outer_classname = "CompressorProto"; @@ -19,10 +17,8 @@ option (udpa.annotations.file_migrate).move_to_package = option (udpa.annotations.file_status).package_version_status = FROZEN; // [#protodoc-title: Compressor] -// Compressor :ref:`configuration overview `. -// [#extension: envoy.filters.http.compressor] -// [#next-free-field: 7] +// [#next-free-field: 6] message Compressor { // Minimum response length, in bytes, which will trigger compression. The default value is 30. google.protobuf.UInt32Value content_length = 1; @@ -49,18 +45,4 @@ message Compressor { // Runtime flag that controls whether the filter is enabled or not. If set to false, the // filter will operate as a pass-through filter. If not specified, defaults to enabled. api.v2.core.RuntimeFeatureFlag runtime_enabled = 5; - - // A compressor library to use for compression. Currently only - // :ref:`envoy.filters.http.compressor.gzip` - // is included in Envoy. - // - // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" - // filter. - CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; -} - -message CompressorLibrary { - // Compressor library specific configuration. See the supported libraries for further - // documentation. - google.protobuf.Any typed_config = 1; } diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD b/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD index f7e871e7b656..ef3541ebcb1d 100644 --- a/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD +++ b/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD @@ -5,8 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = [ - "//envoy/config/filter/http/compressor/gzip/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - ], + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index 362e1c42c5ad..cac7115dceba 100644 --- a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -11,16 +11,13 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.gzip.v3"; option java_outer_classname = "GzipProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; +option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] // [#next-free-field: 6] message Gzip { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.compressor.gzip.v2.Gzip"; - // All the values of this enumeration translate directly to zlib's compression strategies. // For more information about each strategy, please refer to zlib manual. enum CompressionStrategy { diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 635b99d20393..42ed4643caba 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -61,9 +61,6 @@ message Compressor { } message CompressorLibrary { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.compressor.v2.CompressorLibrary"; - // Compressor library specific configuration. See the supported libraries for further // documentation. google.protobuf.Any typed_config = 1; diff --git a/api/envoy/extensions/filters/http/gzip/v3/BUILD b/api/envoy/extensions/filters/http/gzip/v3/BUILD index 1c3cf3aef2b5..c3e93415440e 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/BUILD +++ b/api/envoy/extensions/filters/http/gzip/v3/BUILD @@ -6,8 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/filter/http/compressor/v2:pkg", "//envoy/config/filter/http/gzip/v2:pkg", - "//envoy/extensions/filters/http/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto index eb8a69f083ba..b8399f0e76d3 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.gzip.v3; -import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; +import "envoy/config/filter/http/compressor/v2/compressor.proto"; import "google/protobuf/wrappers.proto"; @@ -75,5 +75,5 @@ message Gzip { // Set of configuration parameters common for all compression filters. If this field is set then // the fields `content_length`, `content_type`, `disable_on_etag_header` and // `remove_accept_encoding_header` are ignored. - compressor.v3.Compressor compressor = 10; + config.filter.http.compressor.v2.Compressor compressor = 10; } diff --git a/api/envoy/extensions/filters/http/gzip/v4alpha/BUILD b/api/envoy/extensions/filters/http/gzip/v4alpha/BUILD new file mode 100644 index 000000000000..7338b943805e --- /dev/null +++ b/api/envoy/extensions/filters/http/gzip/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/filters/http/compressor/v3:pkg", + "//envoy/extensions/filters/http/gzip/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto b/api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto new file mode 100644 index 000000000000..58e7ac78683d --- /dev/null +++ b/api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto @@ -0,0 +1,79 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.gzip.v4alpha; + +import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.gzip.v4alpha"; +option java_outer_classname = "GzipProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Gzip] +// Gzip :ref:`configuration overview `. +// [#extension: envoy.filters.http.gzip] + +// [#next-free-field: 11] +message Gzip { + option (udpa.annotations.versioning).previous_message_type = + "envoy.extensions.filters.http.gzip.v3.Gzip"; + + enum CompressionStrategy { + DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; + } + + message CompressionLevel { + option (udpa.annotations.versioning).previous_message_type = + "envoy.extensions.filters.http.gzip.v3.Gzip.CompressionLevel"; + + enum Enum { + DEFAULT = 0; + BEST = 1; + SPEED = 2; + } + } + + reserved 2, 6, 7, 8; + + reserved "content_length", "content_type", "disable_on_etag_header", + "remove_accept_encoding_header"; + + // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values + // use more memory, but are faster and produce better compression results. The default value is 5. + google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; + + // A value used for selecting the zlib compression level. This setting will affect speed and + // amount of compression applied to the content. "BEST" provides higher compression at the cost of + // higher latency, "SPEED" provides lower compression with minimum impact on response time. + // "DEFAULT" provides an optimal result between speed and compression. This field will be set to + // "DEFAULT" if not specified. + CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + + // A value used for selecting the zlib compression strategy which is directly related to the + // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though + // there are situations which changing this parameter might produce better results. For example, + // run-length encoding (RLE) is typically used when the content is known for having sequences + // which same data occurs many consecutive times. For more information about each strategy, please + // refer to zlib manual. + CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + + // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 12 + // which will produce a 4096 bytes window. For more details about this parameter, please refer to + // zlib manual > deflateInit2. + google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + + // Set of configuration parameters common for all compression filters. If this field is set then + // the fields `content_length`, `content_type`, `disable_on_etag_header` and + // `remove_accept_encoding_header` are ignored. + compressor.v3.Compressor compressor = 10; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index df9410e3e000..cf95d94f2a6b 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -51,6 +51,7 @@ proto_library( "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3alpha:pkg", + "//envoy/extensions/filters/http/compressor/gzip/v3:pkg", "//envoy/extensions/filters/http/compressor/v3:pkg", "//envoy/extensions/filters/http/cors/v3:pkg", "//envoy/extensions/filters/http/csrf/v3:pkg", @@ -160,7 +161,6 @@ proto_library( "//envoy/config/filter/http/aws_request_signing/v2alpha:pkg", "//envoy/config/filter/http/buffer/v2:pkg", "//envoy/config/filter/http/cache/v2alpha:pkg", - "//envoy/config/filter/http/compressor/gzip/v2:pkg", "//envoy/config/filter/http/compressor/v2:pkg", "//envoy/config/filter/http/cors/v2:pkg", "//envoy/config/filter/http/csrf/v2:pkg", diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD deleted file mode 100644 index ef3541ebcb1d..000000000000 --- a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -# DO NOT EDIT. This file is generated by tools/proto_sync.py. - -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], -) diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto b/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto deleted file mode 100644 index 616e55269d7a..000000000000 --- a/generated_api_shadow/envoy/config/filter/http/compressor/gzip/v2/gzip.proto +++ /dev/null @@ -1,81 +0,0 @@ -syntax = "proto3"; - -package envoy.config.filter.http.compressor.gzip.v2; - -import "google/protobuf/wrappers.proto"; - -import "udpa/annotations/migrate.proto"; -import "udpa/annotations/status.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.gzip.v2"; -option java_outer_classname = "GzipProto"; -option java_multiple_files = true; -option (udpa.annotations.file_migrate).move_to_package = - "envoy.extensions.filters.http.compressor.gzip.v3"; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: Gzip] -// [#extension: envoy.filters.http.compressor.gzip] - -// [#next-free-field: 6] -message Gzip { - // All the values of this enumeration translate directly to zlib's compression strategies. - // For more information about each strategy, please refer to zlib manual. - enum CompressionStrategy { - DEFAULT_STRATEGY = 0; - FILTERED = 1; - HUFFMAN_ONLY = 2; - RLE = 3; - FIXED = 4; - } - - enum CompressionLevel { - option allow_alias = true; - - DEFAULT_COMPRESSION = 0; - BEST_SPEED = 1; - COMPRESSION_LEVEL_1 = 1; - COMPRESSION_LEVEL_2 = 2; - COMPRESSION_LEVEL_3 = 3; - COMPRESSION_LEVEL_4 = 4; - COMPRESSION_LEVEL_5 = 5; - COMPRESSION_LEVEL_6 = 6; - COMPRESSION_LEVEL_7 = 7; - COMPRESSION_LEVEL_8 = 8; - COMPRESSION_LEVEL_9 = 9; - BEST_COMPRESSION = 9; - } - - // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values - // use more memory, but are faster and produce better compression results. The default value is 5. - google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; - - // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST_COMPRESSION" provides higher compression - // at the cost of higher latency and is equal to "COMPRESSION_LEVEL_9". "BEST_SPEED" provides - // lower compression with minimum impact on response time, the same as "COMPRESSION_LEVEL_1". - // "DEFAULT_COMPRESSION" provides an optimal result between speed and compression. According - // to zlib's manual this level gives the same result as "COMPRESSION_LEVEL_6". - // This field will be set to "DEFAULT_COMPRESSION" if not specified. - CompressionLevel compression_level = 2 [(validate.rules).enum = {defined_only: true}]; - - // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT_STRATEGY" will be the best choice, - // which is also the default value for the parameter, though there are situations when - // changing this parameter might produce better results. For example, run-length encoding (RLE) - // is typically used when the content is known for having sequences which same data occurs many - // consecutive times. For more information about each strategy, please refer to zlib manual. - CompressionStrategy compression_strategy = 3 [(validate.rules).enum = {defined_only: true}]; - - // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. - // Larger window results in better compression at the expense of memory usage. The default is 12 - // which will produce a 4096 bytes window. For more details about this parameter, please refer to - // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 4 [(validate.rules).uint32 = {lte: 15 gte: 9}]; - - // Value for Zlib's next output buffer. If not set, defaults to 4096. - // See https://www.zlib.net/manual.html for more details. Also see - // https://github.com/envoyproxy/envoy/issues/8448 for context on this filter's performance. - google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; -} diff --git a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto index 69aaacafb762..d62d0d7a42fa 100644 --- a/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto +++ b/generated_api_shadow/envoy/config/filter/http/compressor/v2/compressor.proto @@ -4,25 +4,21 @@ package envoy.config.filter.http.compressor.v2; import "envoy/api/v2/core/base.proto"; -import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; -import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.filter.http.compressor.v2"; option java_outer_classname = "CompressorProto"; option java_multiple_files = true; option (udpa.annotations.file_migrate).move_to_package = "envoy.extensions.filters.http.compressor.v3"; -option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (udpa.annotations.file_status).package_version_status = FROZEN; // [#protodoc-title: Compressor] -// Compressor :ref:`configuration overview `. -// [#extension: envoy.filters.http.compressor] -// [#next-free-field: 7] +// [#next-free-field: 6] message Compressor { // Minimum response length, in bytes, which will trigger compression. The default value is 30. google.protobuf.UInt32Value content_length = 1; @@ -49,18 +45,4 @@ message Compressor { // Runtime flag that controls whether the filter is enabled or not. If set to false, the // filter will operate as a pass-through filter. If not specified, defaults to enabled. api.v2.core.RuntimeFeatureFlag runtime_enabled = 5; - - // A compressor library to use for compression. Currently only - // :ref:`envoy.filters.http.compressor.gzip` - // is included in Envoy. - // - // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" - // filter. - CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; -} - -message CompressorLibrary { - // Compressor library specific configuration. See the supported libraries for further - // documentation. - google.protobuf.Any typed_config = 1; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD index f7e871e7b656..ef3541ebcb1d 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD @@ -5,8 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = [ - "//envoy/config/filter/http/compressor/gzip/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - ], + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], ) diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto index 362e1c42c5ad..cac7115dceba 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto @@ -11,16 +11,13 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.gzip.v3"; option java_outer_classname = "GzipProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; +option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] // [#extension: envoy.filters.http.compressor.gzip] // [#next-free-field: 6] message Gzip { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.compressor.gzip.v2.Gzip"; - // All the values of this enumeration translate directly to zlib's compression strategies. // For more information about each strategy, please refer to zlib manual. enum CompressionStrategy { diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 71161d17f131..42ed4643caba 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -14,7 +14,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.v3"; option java_outer_classname = "CompressorProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; +option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Compressor] // Compressor :ref:`configuration overview `. @@ -61,9 +61,6 @@ message Compressor { } message CompressorLibrary { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.compressor.v2.CompressorLibrary"; - // Compressor library specific configuration. See the supported libraries for further // documentation. google.protobuf.Any typed_config = 1; diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD index 1c3cf3aef2b5..c3e93415440e 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD @@ -6,8 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/filter/http/compressor/v2:pkg", "//envoy/config/filter/http/gzip/v2:pkg", - "//envoy/extensions/filters/http/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto index 3206037723de..43859be8165c 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.gzip.v3; -import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; +import "envoy/config/filter/http/compressor/v2/compressor.proto"; import "google/protobuf/wrappers.proto"; @@ -70,7 +70,7 @@ message Gzip { // Set of configuration parameters common for all compression filters. If this field is set then // the fields `content_length`, `content_type`, `disable_on_etag_header` and // `remove_accept_encoding_header` are ignored. - compressor.v3.Compressor compressor = 10; + config.filter.http.compressor.v2.Compressor compressor = 10; google.protobuf.UInt32Value hidden_envoy_deprecated_content_length = 2 [deprecated = true]; diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD new file mode 100644 index 000000000000..7338b943805e --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/filters/http/compressor/v3:pkg", + "//envoy/extensions/filters/http/gzip/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto new file mode 100644 index 000000000000..58e7ac78683d --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto @@ -0,0 +1,79 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.gzip.v4alpha; + +import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.gzip.v4alpha"; +option java_outer_classname = "GzipProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Gzip] +// Gzip :ref:`configuration overview `. +// [#extension: envoy.filters.http.gzip] + +// [#next-free-field: 11] +message Gzip { + option (udpa.annotations.versioning).previous_message_type = + "envoy.extensions.filters.http.gzip.v3.Gzip"; + + enum CompressionStrategy { + DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; + } + + message CompressionLevel { + option (udpa.annotations.versioning).previous_message_type = + "envoy.extensions.filters.http.gzip.v3.Gzip.CompressionLevel"; + + enum Enum { + DEFAULT = 0; + BEST = 1; + SPEED = 2; + } + } + + reserved 2, 6, 7, 8; + + reserved "content_length", "content_type", "disable_on_etag_header", + "remove_accept_encoding_header"; + + // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values + // use more memory, but are faster and produce better compression results. The default value is 5. + google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; + + // A value used for selecting the zlib compression level. This setting will affect speed and + // amount of compression applied to the content. "BEST" provides higher compression at the cost of + // higher latency, "SPEED" provides lower compression with minimum impact on response time. + // "DEFAULT" provides an optimal result between speed and compression. This field will be set to + // "DEFAULT" if not specified. + CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; + + // A value used for selecting the zlib compression strategy which is directly related to the + // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though + // there are situations which changing this parameter might produce better results. For example, + // run-length encoding (RLE) is typically used when the content is known for having sequences + // which same data occurs many consecutive times. For more information about each strategy, please + // refer to zlib manual. + CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; + + // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 12 + // which will produce a 4096 bytes window. For more details about this parameter, please refer to + // zlib manual > deflateInit2. + google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; + + // Set of configuration parameters common for all compression filters. If this field is set then + // the fields `content_length`, `content_type`, `disable_on_etag_header` and + // `remove_accept_encoding_header` are ignored. + compressor.v3.Compressor compressor = 10; +} diff --git a/source/extensions/filters/http/gzip/BUILD b/source/extensions/filters/http/gzip/BUILD index 5b1f7517b66f..168783406be1 100644 --- a/source/extensions/filters/http/gzip/BUILD +++ b/source/extensions/filters/http/gzip/BUILD @@ -21,6 +21,7 @@ envoy_cc_library( "//source/common/http:headers_lib", "//source/common/protobuf", "//source/extensions/filters/http/common/compressor:compressor_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/gzip/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index 846500cd6dab..2e85a21385d5 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -1,5 +1,7 @@ #include "extensions/filters/http/gzip/gzip_filter.h" +#include "envoy/config/core/v3/base.pb.h" + #include "common/http/headers.h" #include "common/protobuf/protobuf.h" @@ -72,10 +74,28 @@ uint64_t GzipFilterConfig::windowBitsUint(Protobuf::uint32 window_bits) { const envoy::extensions::filters::http::compressor::v3::Compressor GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip::v3::Gzip& gzip) { + envoy::extensions::filters::http::compressor::v3::Compressor compressor = {}; + + // The deprecated gzip filter is using v2 Compressor field to avoid setting the + // compressor_library field which is mandatory in v3. Here we convert v2 Compressor to v3. if (gzip.has_compressor()) { - return gzip.compressor(); + compressor.set_allocated_content_length( + new Protobuf::UInt32Value(gzip.compressor().content_length())); + for (const std::string& ctype : gzip.compressor().content_type()) { + compressor.add_content_type(ctype); + } + compressor.set_disable_on_etag_header(gzip.compressor().disable_on_etag_header()); + compressor.set_remove_accept_encoding_header(gzip.compressor().remove_accept_encoding_header()); + if (gzip.compressor().has_runtime_enabled()) { + auto feature_flag(new envoy::config::core::v3::RuntimeFeatureFlag()); + feature_flag->set_runtime_key(gzip.compressor().runtime_enabled().runtime_key()); + feature_flag->set_allocated_default_value( + new Protobuf::BoolValue(gzip.compressor().runtime_enabled().default_value())); + compressor.set_allocated_runtime_enabled(feature_flag); + } + return compressor; } - envoy::extensions::filters::http::compressor::v3::Compressor compressor = {}; + if (gzip.has_hidden_envoy_deprecated_content_length()) { compressor.set_allocated_content_length( new Protobuf::UInt32Value(gzip.hidden_envoy_deprecated_content_length())); diff --git a/test/extensions/filters/http/common/compressor/BUILD b/test/extensions/filters/http/common/compressor/BUILD index b03a3cf39122..e08df739e012 100644 --- a/test/extensions/filters/http/common/compressor/BUILD +++ b/test/extensions/filters/http/common/compressor/BUILD @@ -14,6 +14,7 @@ envoy_cc_test( deps = [ "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common/compressor:compressor_lib", + "//source/extensions/filters/http/compressor/gzip:config", "//test/mocks/http:http_mocks", "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index a82c86444d0a..b8fe47736e91 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -47,7 +47,17 @@ class CompressorFilterTest : public testing::Test { .WillByDefault(Return(true)); } - void SetUp() override { setUpFilter("{}"); } + void SetUp() override { + setUpFilter(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF"); + } // CompressorFilter private member functions void sanitizeEtagHeader(Http::ResponseHeaderMap& headers) { @@ -174,6 +184,11 @@ TEST_F(CompressorFilterTest, DecodeHeadersWithRuntimeDisabled) { "runtime_enabled": { "default_value": true, "runtime_key": "foo_key" + }, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } } } )EOF"); @@ -351,7 +366,16 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { Stats::TestUtil::TestStore stats; NiceMock runtime; envoy::extensions::filters::http::compressor::v3::Compressor compressor; - TestUtility::loadFromJson("{}", compressor); + TestUtility::loadFromJson(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF", + compressor); CompressorFilterConfigSharedPtr config2; config2 = std::make_shared(compressor, "test2.", stats, runtime, "test2"); @@ -375,7 +399,16 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { ; NiceMock runtime; envoy::extensions::filters::http::compressor::v3::Compressor compressor; - TestUtility::loadFromJson("{}", compressor); + TestUtility::loadFromJson(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF", + compressor); CompressorFilterConfigSharedPtr config2; config2 = std::make_shared(compressor, "test2.", stats, runtime, "gzip"); @@ -395,7 +428,16 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { ; NiceMock runtime; envoy::extensions::filters::http::compressor::v3::Compressor compressor; - TestUtility::loadFromJson("{}", compressor); + TestUtility::loadFromJson(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF", + compressor); CompressorFilterConfigSharedPtr config2; config2 = std::make_shared(compressor, "test2.", stats, runtime, "test"); @@ -415,7 +457,16 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { ; NiceMock runtime; envoy::extensions::filters::http::compressor::v3::Compressor compressor; - TestUtility::loadFromJson("{}", compressor); + TestUtility::loadFromJson(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF", + compressor); CompressorFilterConfigSharedPtr config2; config2 = std::make_shared(compressor, "test2.", stats, runtime, "test"); @@ -435,7 +486,16 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { ; NiceMock runtime; envoy::extensions::filters::http::compressor::v3::Compressor compressor; - TestUtility::loadFromJson("{}", compressor); + TestUtility::loadFromJson(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF", + compressor); CompressorFilterConfigSharedPtr config1; config1 = std::make_shared(compressor, "test1.", stats, runtime, "test1"); @@ -465,7 +525,16 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { ; NiceMock runtime; envoy::extensions::filters::http::compressor::v3::Compressor compressor; - TestUtility::loadFromJson("{}", compressor); + TestUtility::loadFromJson(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF", + compressor); CompressorFilterConfigSharedPtr config1; config1 = std::make_shared(compressor, "test1.", stats, runtime, "test1"); @@ -517,7 +586,16 @@ TEST_F(CompressorFilterTest, IsMinimumContentLength) { EXPECT_TRUE(isMinimumContentLength(headers)); } - setUpFilter(R"EOF({"content_length": 500})EOF"); + setUpFilter(R"EOF( +{ + "content_length": 500, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF"); { Http::TestResponseHeaderMapImpl headers = {{"content-length", "501"}}; EXPECT_TRUE(isMinimumContentLength(headers)); @@ -540,7 +618,16 @@ TEST_F(CompressorFilterTest, ContentLengthNoCompression) { // Verifies that compression is NOT skipped when content-length header is allowed. TEST_F(CompressorFilterTest, ContentLengthCompression) { - setUpFilter(R"EOF({"content_length": 500})EOF"); + setUpFilter(R"EOF( +{ + "content_length": 500, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF"); doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); doResponseCompression({{":method", "get"}, {"content-length", "1000"}}, false); } @@ -603,7 +690,12 @@ TEST_F(CompressorFilterTest, IsContentTypeAllowed) { "text/html", "xyz/svg+xml", "Test/INSENSITIVE" - ] + ], + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } } )EOF"); @@ -641,7 +733,12 @@ TEST_F(CompressorFilterTest, ContentTypeNoCompression) { "application/json", "font/eot", "image/svg+xml" - ] + ], + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } } )EOF"); doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); @@ -698,7 +795,16 @@ TEST_F(CompressorFilterTest, IsEtagAllowed) { EXPECT_EQ(0, stats_.counter("test.test.not_compressed_etag").value()); } - setUpFilter(R"EOF({ "disable_on_etag_header": true })EOF"); + setUpFilter(R"EOF( +{ + "disable_on_etag_header": true, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF"); { Http::TestResponseHeaderMapImpl headers = {{"etag", R"EOF(W/"686897696a7c876b7e")EOF"}}; EXPECT_FALSE(isEtagAllowed(headers)); @@ -718,7 +824,16 @@ TEST_F(CompressorFilterTest, IsEtagAllowed) { // Verifies that compression is skipped when etag header is NOT allowed. TEST_F(CompressorFilterTest, EtagNoCompression) { - setUpFilter(R"EOF({ "disable_on_etag_header": true })EOF"); + setUpFilter(R"EOF( +{ + "disable_on_etag_header": true, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF"); doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); doResponseNoCompression( {{":method", "get"}, {"content-length", "256"}, {"etag", R"EOF(W/"686897696a7c876b7e")EOF"}}); @@ -886,13 +1001,30 @@ TEST_F(CompressorFilterTest, RemoveAcceptEncodingHeader) { filter_->setDecoderFilterCallbacks(decoder_callbacks); { Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; - setUpFilter(R"EOF({"remove_accept_encoding_header": true})EOF"); + setUpFilter(R"EOF( +{ + "remove_accept_encoding_header": true, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF"); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); EXPECT_FALSE(headers.has("accept-encoding")); } { Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; - setUpFilter("{}"); + setUpFilter(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + } + } +} +)EOF"); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); EXPECT_TRUE(headers.has("accept-encoding")); EXPECT_EQ("deflate, test, gzip, br", headers.get_("accept-encoding")); diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 25af409f28fe..b1c9014c2fbb 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -65,24 +65,6 @@ class CompressorIntegrationTest : public testing::TestWithParambody(), std::string(content_length, 'a')); } - const std::string deprecated_full_config{R"EOF( - name: envoy.filters.http.compressor - typed_config: - "@type": type.googleapis.com/envoy.config.filter.http.compressor.v2.Compressor - disable_on_etag_header: true - content_length: 100 - content_type: - - text/html - - application/json - compressor_library: - typed_config: - "@type": type.googleapis.com/envoy.config.filter.http.compressor.gzip.v2.Gzip - memory_level: 3 - window_bits: 10 - compression_level: best_compression - compression_strategy: rle - )EOF"}; - const std::string full_config{R"EOF( name: compressor typed_config: @@ -134,21 +116,6 @@ TEST_P(CompressorIntegrationTest, AcceptanceDefaultConfigTest) { {"content-type", "text/xml"}}); } -/** - * Exercises gzip compression with deprecated full configuration. - */ -TEST_P(CompressorIntegrationTest, AcceptanceDeprecatedFullConfigTest) { - initializeFilter(deprecated_full_config); - doRequestAndCompression(Http::TestRequestHeaderMapImpl{{":method", "GET"}, - {":path", "/test/long/url"}, - {":scheme", "http"}, - {":authority", "host"}, - {"accept-encoding", "deflate, gzip"}}, - Http::TestResponseHeaderMapImpl{{":status", "200"}, - {"content-length", "4400"}, - {"content-type", "application/json"}}); -} - /** * Exercises gzip compression with full configuration. */ diff --git a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc index 28cb3b9a21f8..6b77e67d5b11 100644 --- a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc @@ -124,7 +124,7 @@ class GzipCompressorFilterTest : public testing::Test { Decompressor::ZlibDecompressorImpl decompressor_; Buffer::OwnedImpl decompressed_data_; std::string expected_str_; - Stats::IsolatedStoreImpl stats_; + Stats::TestUtil::TestStore stats_; NiceMock runtime_; NiceMock encoder_callbacks_; NiceMock decoder_callbacks_; From 9bcc46952bfd6268d0c93c6d026594801f96673b Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 14 Apr 2020 11:39:11 +0300 Subject: [PATCH 15/56] update version history Signed-off-by: Dmitry Rozhkov --- docs/root/version_history/current.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index d1339aa24f5f..a2f2a748f3cb 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -4,6 +4,7 @@ Changes ------- +* compressor: generic :ref:`compressor ` filter exposed to users. * http: fixed a bug where the upgrade header was not cleared on responses to non-upgrade requests. Can be reverted temporarily by setting runtime feature `envoy.reloadable_features.fix_upgrade_response` to false. * tracing: tracing configuration has been made fully dynamic and every HTTP connection manager @@ -15,3 +16,5 @@ Deprecated * Tracing provider configuration as part of :ref:`bootstrap config ` has been deprecated in favor of configuration as part of :ref:`HTTP connection manager `. +* The :ref:`HTTP Gzip filter ` has been deprecated in favor of + :ref:`Compressor `. From 21fb4ebb38f197355baf1dca1e102bb9fee504ef Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 14 Apr 2020 12:57:10 +0300 Subject: [PATCH 16/56] fix pedantic spelling check Signed-off-by: Dmitry Rozhkov --- source/common/compressor/zlib_compressor_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 81206a4236af..2610ea0af775 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -30,7 +30,7 @@ class ZlibCompressorImpl : public Zlib::Base, public Compressor { * Enum values used to set compression level during initialization. * best: gives best compression. * speed: gives best performance. - * levelX: allows to adjust tradoffs more precisely - from level1 (best speed, but very + * levelX: allows to adjust trad-offs more precisely - from level1 (best speed, but very * low compression ratio) to level9 (best compression, but low speed). * standard: requests a default compromise between speed and compression. (default) @see zlib * manual. From 2f485488217eb971fc9d234c3f01070f4ecf686d Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 14 Apr 2020 13:26:25 +0300 Subject: [PATCH 17/56] make use of PROTOBUF_GET_WRAPPED_OR_DEFAULT Signed-off-by: Dmitry Rozhkov --- .../filters/http/compressor/gzip/config.cc | 17 ++++++----------- .../filters/http/compressor/gzip/config.h | 3 --- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/source/extensions/filters/http/compressor/gzip/config.cc b/source/extensions/filters/http/compressor/gzip/config.cc index d9db5830702b..f38585f60c91 100644 --- a/source/extensions/filters/http/compressor/gzip/config.cc +++ b/source/extensions/filters/http/compressor/gzip/config.cc @@ -16,15 +16,18 @@ const uint64_t DefaultWindowBits = 12; // When summed to window bits, this sets a gzip header and trailer around the compressed data. const uint64_t GzipHeaderValue = 16; +// Default zlib chunk size. +const uint32_t DefaultChunkSize = 4096; } // namespace GzipCompressorFactory::GzipCompressorFactory( const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& gzip) : compression_level_(compressionLevelEnum(gzip.compression_level())), compression_strategy_(compressionStrategyEnum(gzip.compression_strategy())), - memory_level_(memoryLevelUint(gzip.memory_level().value())), - window_bits_(windowBitsUint(gzip.window_bits().value())), - chunk_size_(gzip.chunk_size().value() > 0 ? gzip.chunk_size().value() : 4096) {} + memory_level_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(gzip, memory_level, DefaultMemoryLevel)), + window_bits_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(gzip, window_bits, DefaultWindowBits) | + GzipHeaderValue), + chunk_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(gzip, chunk_size, DefaultChunkSize)) {} Envoy::Compressor::ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::compressionLevelEnum( envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel @@ -71,14 +74,6 @@ GzipCompressorFactory::compressionStrategyEnum( } } -uint64_t GzipCompressorFactory::memoryLevelUint(Protobuf::uint32 level) { - return level > 0 ? level : DefaultMemoryLevel; -} - -uint64_t GzipCompressorFactory::windowBitsUint(Protobuf::uint32 window_bits) { - return (window_bits > 0 ? window_bits : DefaultWindowBits) | GzipHeaderValue; -} - std::unique_ptr GzipCompressorFactory::createCompressor() { auto compressor = std::make_unique(chunk_size_); compressor->init(compression_level_, compression_strategy_, window_bits_, memory_level_); diff --git a/source/extensions/filters/http/compressor/gzip/config.h b/source/extensions/filters/http/compressor/gzip/config.h index 5f93612725b2..0355c902845c 100644 --- a/source/extensions/filters/http/compressor/gzip/config.h +++ b/source/extensions/filters/http/compressor/gzip/config.h @@ -44,9 +44,6 @@ class GzipCompressorFactory : public Compressor::CompressorFactory { envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionStrategy compression_strategy); - static uint64_t memoryLevelUint(Protobuf::uint32 level); - static uint64_t windowBitsUint(Protobuf::uint32 window_bits); - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compression_level_; Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compression_strategy_; From 63085ac5060269b7b6bca0c1648ef14c71b7f8a8 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 14 Apr 2020 13:50:55 +0300 Subject: [PATCH 18/56] compressor: introduce CompressorPtr type alias Signed-off-by: Dmitry Rozhkov --- include/envoy/compressor/compressor.h | 2 ++ source/extensions/filters/http/common/compressor/compressor.h | 4 ++-- .../extensions/filters/http/compressor/compressor_factory.h | 2 +- .../extensions/filters/http/compressor/compressor_filter.cc | 2 +- source/extensions/filters/http/compressor/compressor_filter.h | 2 +- source/extensions/filters/http/compressor/gzip/config.cc | 2 +- source/extensions/filters/http/compressor/gzip/config.h | 2 +- source/extensions/filters/http/gzip/gzip_filter.cc | 2 +- source/extensions/filters/http/gzip/gzip_filter.h | 2 +- .../filters/http/common/compressor/compressor_filter_test.cc | 4 +--- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/envoy/compressor/compressor.h b/include/envoy/compressor/compressor.h index d25204a3ead4..2ffe63efcad9 100644 --- a/include/envoy/compressor/compressor.h +++ b/include/envoy/compressor/compressor.h @@ -26,5 +26,7 @@ class Compressor { virtual void compress(Buffer::Instance& buffer, State state) PURE; }; +using CompressorPtr = std::unique_ptr; + } // namespace Compressor } // namespace Envoy diff --git a/source/extensions/filters/http/common/compressor/compressor.h b/source/extensions/filters/http/common/compressor/compressor.h index 8972a2d0db3e..4a15782b8312 100644 --- a/source/extensions/filters/http/common/compressor/compressor.h +++ b/source/extensions/filters/http/common/compressor/compressor.h @@ -62,7 +62,7 @@ class CompressorFilterConfig { CompressorFilterConfig() = delete; virtual ~CompressorFilterConfig() = default; - virtual std::unique_ptr makeCompressor() PURE; + virtual Envoy::Compressor::CompressorPtr makeCompressor() PURE; bool enabled() const { return enabled_.enabled(); } const CompressorStats& stats() { return stats_; } @@ -149,7 +149,7 @@ class CompressorFilter : public Http::PassThroughFilter { bool shouldCompress(const EncodingDecision& decision) const; bool skip_compression_; - std::unique_ptr compressor_; + Envoy::Compressor::CompressorPtr compressor_; const CompressorFilterConfigSharedPtr config_; std::unique_ptr accept_encoding_; }; diff --git a/source/extensions/filters/http/compressor/compressor_factory.h b/source/extensions/filters/http/compressor/compressor_factory.h index 481e5999671e..229594999830 100644 --- a/source/extensions/filters/http/compressor/compressor_factory.h +++ b/source/extensions/filters/http/compressor/compressor_factory.h @@ -11,7 +11,7 @@ class CompressorFactory { public: virtual ~CompressorFactory() = default; - virtual std::unique_ptr createCompressor() PURE; + virtual Envoy::Compressor::CompressorPtr createCompressor() PURE; virtual const std::string& statsPrefix() const PURE; virtual const std::string& contentEncoding() const PURE; }; diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index ef5436b76719..e70f942ea911 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -14,7 +14,7 @@ CompressorFilterConfig::CompressorFilterConfig( compressor_factory->contentEncoding()), compressor_factory_(std::move(compressor_factory)) {} -std::unique_ptr CompressorFilterConfig::makeCompressor() { +Envoy::Compressor::CompressorPtr CompressorFilterConfig::makeCompressor() { return compressor_factory_->createCompressor(); } diff --git a/source/extensions/filters/http/compressor/compressor_filter.h b/source/extensions/filters/http/compressor/compressor_filter.h index 0ac1322ea972..74c82aa7071c 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.h +++ b/source/extensions/filters/http/compressor/compressor_filter.h @@ -23,7 +23,7 @@ class CompressorFilterConfig : public Common::Compressors::CompressorFilterConfi const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, CompressorFactoryPtr compressor_factory); - std::unique_ptr makeCompressor() override; + Envoy::Compressor::CompressorPtr makeCompressor() override; private: const CompressorFactoryPtr compressor_factory_; diff --git a/source/extensions/filters/http/compressor/gzip/config.cc b/source/extensions/filters/http/compressor/gzip/config.cc index f38585f60c91..2ef72223a220 100644 --- a/source/extensions/filters/http/compressor/gzip/config.cc +++ b/source/extensions/filters/http/compressor/gzip/config.cc @@ -74,7 +74,7 @@ GzipCompressorFactory::compressionStrategyEnum( } } -std::unique_ptr GzipCompressorFactory::createCompressor() { +Envoy::Compressor::CompressorPtr GzipCompressorFactory::createCompressor() { auto compressor = std::make_unique(chunk_size_); compressor->init(compression_level_, compression_strategy_, window_bits_, memory_level_); return compressor; diff --git a/source/extensions/filters/http/compressor/gzip/config.h b/source/extensions/filters/http/compressor/gzip/config.h index 0355c902845c..11fe02447c74 100644 --- a/source/extensions/filters/http/compressor/gzip/config.h +++ b/source/extensions/filters/http/compressor/gzip/config.h @@ -28,7 +28,7 @@ class GzipCompressorFactory : public Compressor::CompressorFactory { public: GzipCompressorFactory(const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& gzip); - std::unique_ptr createCompressor() override; + Envoy::Compressor::CompressorPtr createCompressor() override; const std::string& statsPrefix() const override { return gzipStatsPrefix(); } const std::string& contentEncoding() const override { return Http::Headers::get().ContentEncodingValues.Gzip; diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index 2e85a21385d5..f3803c914fed 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -32,7 +32,7 @@ GzipFilterConfig::GzipFilterConfig(const envoy::extensions::filters::http::gzip: memory_level_(memoryLevelUint(gzip.memory_level().value())), window_bits_(windowBitsUint(gzip.window_bits().value())) {} -std::unique_ptr GzipFilterConfig::makeCompressor() { +Compressor::CompressorPtr GzipFilterConfig::makeCompressor() { auto compressor = std::make_unique(); compressor->init(compressionLevel(), compressionStrategy(), windowBits(), memoryLevel()); return compressor; diff --git a/source/extensions/filters/http/gzip/gzip_filter.h b/source/extensions/filters/http/gzip/gzip_filter.h index a7c6406c2dc4..a996dd4f7937 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.h +++ b/source/extensions/filters/http/gzip/gzip_filter.h @@ -20,7 +20,7 @@ class GzipFilterConfig : public Common::Compressors::CompressorFilterConfig { GzipFilterConfig(const envoy::extensions::filters::http::gzip::v3::Gzip& gzip, const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime); - std::unique_ptr makeCompressor() override; + Compressor::CompressorPtr makeCompressor() override; Compressor::ZlibCompressorImpl::CompressionLevel compressionLevel() const { return compression_level_; diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index b8fe47736e91..60e9346d5fdd 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -35,9 +35,7 @@ class MockCompressorFilterConfig : public CompressorFilterConfig { : CompressorFilterConfig(compressor, stats_prefix + compressor_name + ".", scope, runtime, compressor_name) {} - std::unique_ptr makeCompressor() override { - return std::make_unique(); - } + Compressor::CompressorPtr makeCompressor() override { return std::make_unique(); } }; class CompressorFilterTest : public testing::Test { From edbe6b52f86170dcdcb0e2034592943b3ff999ba Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 14 Apr 2020 14:13:41 +0300 Subject: [PATCH 19/56] rename createCompressorLibraryFromProto to createCompressorFactoryFromProto Signed-off-by: Dmitry Rozhkov --- .../filters/http/compressor/compressor_library_config.h | 2 +- source/extensions/filters/http/compressor/config.cc | 2 +- source/extensions/filters/http/compressor/factory_base.h | 6 +++--- source/extensions/filters/http/compressor/gzip/config.cc | 4 ++-- source/extensions/filters/http/compressor/gzip/config.h | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/extensions/filters/http/compressor/compressor_library_config.h b/source/extensions/filters/http/compressor/compressor_library_config.h index a4dcdaad2659..fb6d7fa822d4 100644 --- a/source/extensions/filters/http/compressor/compressor_library_config.h +++ b/source/extensions/filters/http/compressor/compressor_library_config.h @@ -15,7 +15,7 @@ class NamedCompressorLibraryConfigFactory : public Config::TypedFactory { ~NamedCompressorLibraryConfigFactory() override = default; virtual CompressorFactoryPtr - createCompressorLibraryFromProto(const Protobuf::Message& config, + createCompressorFactoryFromProto(const Protobuf::Message& config, Server::Configuration::FactoryContext& context) PURE; }; diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc index ade20f1e56d7..ff95006c1cb6 100644 --- a/source/extensions/filters/http/compressor/config.cc +++ b/source/extensions/filters/http/compressor/config.cc @@ -26,7 +26,7 @@ Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped proto_config.compressor_library().typed_config(), context.messageValidationVisitor(), *config_factory); CompressorFactoryPtr compressor_factory = - config_factory->createCompressorLibraryFromProto(*message, context); + config_factory->createCompressorFactoryFromProto(*message, context); Common::Compressors::CompressorFilterConfigSharedPtr config = std::make_shared(proto_config, stats_prefix, context.scope(), context.runtime(), std::move(compressor_factory)); diff --git a/source/extensions/filters/http/compressor/factory_base.h b/source/extensions/filters/http/compressor/factory_base.h index ce564006c708..cc3ec0e52925 100644 --- a/source/extensions/filters/http/compressor/factory_base.h +++ b/source/extensions/filters/http/compressor/factory_base.h @@ -14,9 +14,9 @@ template class CompressorLibraryFactoryBase : public NamedCompressorLibraryConfigFactory { public: CompressorFactoryPtr - createCompressorLibraryFromProto(const Protobuf::Message& proto_config, + createCompressorFactoryFromProto(const Protobuf::Message& proto_config, Server::Configuration::FactoryContext& context) override { - return createCompressorLibraryFromProtoTyped( + return createCompressorFactoryFromProtoTyped( MessageUtil::downcastAndValidate(proto_config, context.messageValidationVisitor())); } @@ -33,7 +33,7 @@ class CompressorLibraryFactoryBase : public NamedCompressorLibraryConfigFactory CompressorLibraryFactoryBase(const std::string& name) : name_(name) {} private: - virtual CompressorFactoryPtr createCompressorLibraryFromProtoTyped(const ConfigProto&) PURE; + virtual CompressorFactoryPtr createCompressorFactoryFromProtoTyped(const ConfigProto&) PURE; const std::string name_; }; diff --git a/source/extensions/filters/http/compressor/gzip/config.cc b/source/extensions/filters/http/compressor/gzip/config.cc index 2ef72223a220..4336f5578f96 100644 --- a/source/extensions/filters/http/compressor/gzip/config.cc +++ b/source/extensions/filters/http/compressor/gzip/config.cc @@ -81,13 +81,13 @@ Envoy::Compressor::CompressorPtr GzipCompressorFactory::createCompressor() { } Compressor::CompressorFactoryPtr -GzipCompressorLibraryFactory::createCompressorLibraryFromProtoTyped( +GzipCompressorLibraryFactory::createCompressorFactoryFromProtoTyped( const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& proto_config) { return std::make_unique(proto_config); } /** - * Static registration for the gzip filter. @see NamedCompressorLibraryConfigFactory. + * Static registration for the gzip compressor library. @see NamedCompressorLibraryConfigFactory. */ REGISTER_FACTORY(GzipCompressorLibraryFactory, NamedCompressorLibraryConfigFactory); diff --git a/source/extensions/filters/http/compressor/gzip/config.h b/source/extensions/filters/http/compressor/gzip/config.h index 11fe02447c74..798181f91a12 100644 --- a/source/extensions/filters/http/compressor/gzip/config.h +++ b/source/extensions/filters/http/compressor/gzip/config.h @@ -59,7 +59,7 @@ class GzipCompressorLibraryFactory GzipCompressorLibraryFactory() : CompressorLibraryFactoryBase(gzipExtensionName()) {} private: - Compressor::CompressorFactoryPtr createCompressorLibraryFromProtoTyped( + Compressor::CompressorFactoryPtr createCompressorFactoryFromProtoTyped( const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& config) override; }; From da5a3055f4a9009795edf3aeb9e1a23a494234dc Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 14 Apr 2020 14:36:57 +0300 Subject: [PATCH 20/56] remove unused header Signed-off-by: Dmitry Rozhkov --- source/extensions/filters/http/compressor/BUILD | 1 - source/extensions/filters/http/compressor/config.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/source/extensions/filters/http/compressor/BUILD b/source/extensions/filters/http/compressor/BUILD index 3230e998f159..2b79be2a8b22 100644 --- a/source/extensions/filters/http/compressor/BUILD +++ b/source/extensions/filters/http/compressor/BUILD @@ -62,7 +62,6 @@ envoy_cc_extension( "//source/common/config:utility_lib", "//source/extensions/filters/http:well_known_names", "//source/extensions/filters/http/common:factory_base_lib", - "//source/extensions/filters/http/compressor/gzip:config", "@envoy_api//envoy/extensions/filters/http/compressor/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc index ff95006c1cb6..1e7608e40412 100644 --- a/source/extensions/filters/http/compressor/config.cc +++ b/source/extensions/filters/http/compressor/config.cc @@ -4,7 +4,6 @@ #include "extensions/filters/http/compressor/compressor_filter.h" #include "extensions/filters/http/compressor/compressor_library_config.h" -#include "extensions/filters/http/compressor/gzip/config.h" namespace Envoy { namespace Extensions { From 76c2b1fa1f96469c0452dd51d5ffb25498cd79ca Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 14 Apr 2020 15:40:14 +0300 Subject: [PATCH 21/56] docs: fix references Signed-off-by: Dmitry Rozhkov --- docs/root/api-v2/config/filter/http/http.rst | 1 - .../configuration/http/http_filters/compressor_filter.rst | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/root/api-v2/config/filter/http/http.rst b/docs/root/api-v2/config/filter/http/http.rst index b54c56b28e41..7746b0d72a5c 100644 --- a/docs/root/api-v2/config/filter/http/http.rst +++ b/docs/root/api-v2/config/filter/http/http.rst @@ -9,4 +9,3 @@ HTTP filters */v2/* */v2alpha/* */v2alpha1/* - compressor/*/v2/* diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst index d4eeee2dc140..26c70b84aa53 100644 --- a/docs/root/configuration/http/http_filters/compressor_filter.rst +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -13,7 +13,7 @@ at the expense of higher CPU load or offloading it to a compression accelerator. Configuration ------------- -* :ref:`v2 API reference ` +* :ref:`v3 API reference ` * This filter should be configured with the name *envoy.filters.http.compressor*. How it works @@ -23,7 +23,7 @@ determine whether or not the content should be compressed. The content is compressed and then sent to the client with the appropriate headers, if response and request allow. -Currently the filter supports :ref:`gzip compression` +Currently the filter supports :ref:`gzip compression ` only. Other compression libraries can be supported as extensions. An example configuration of the filter may look like the following: From bc086c0a7e441c6adbc926dcbfffa4d64cb78691 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 15 Apr 2020 10:34:18 +0300 Subject: [PATCH 22/56] avoid using v2 types in v3 configs Signed-off-by: Dmitry Rozhkov --- .../http/compressor/v3/compressor.proto | 3 - .../extensions/filters/http/gzip/v3/BUILD | 2 +- .../filters/http/gzip/v3/gzip.proto | 33 +++++++- .../filters/http/gzip/v4alpha/BUILD | 13 --- .../filters/http/gzip/v4alpha/gzip.proto | 79 ------------------- .../http/compressor/v3/compressor.proto | 3 - .../extensions/filters/http/gzip/v3/BUILD | 2 +- .../filters/http/gzip/v3/gzip.proto | 33 +++++++- .../filters/http/gzip/v4alpha/BUILD | 13 --- .../filters/http/gzip/v4alpha/gzip.proto | 79 ------------------- 10 files changed, 64 insertions(+), 196 deletions(-) delete mode 100644 api/envoy/extensions/filters/http/gzip/v4alpha/BUILD delete mode 100644 api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto delete mode 100644 generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD delete mode 100644 generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 42ed4643caba..c6d123acd2c1 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -54,9 +54,6 @@ message Compressor { // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. - // - // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" - // filter. CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } diff --git a/api/envoy/extensions/filters/http/gzip/v3/BUILD b/api/envoy/extensions/filters/http/gzip/v3/BUILD index c3e93415440e..6bd8f8337ac8 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/BUILD +++ b/api/envoy/extensions/filters/http/gzip/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/config/filter/http/compressor/v2:pkg", + "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/gzip/v2:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], diff --git a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto index b8399f0e76d3..5bdbfcaab521 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.gzip.v3; -import "envoy/config/filter/http/compressor/v2/compressor.proto"; +import "envoy/config/core/v3/base.proto"; import "google/protobuf/wrappers.proto"; @@ -75,5 +75,34 @@ message Gzip { // Set of configuration parameters common for all compression filters. If this field is set then // the fields `content_length`, `content_type`, `disable_on_etag_header` and // `remove_accept_encoding_header` are ignored. - config.filter.http.compressor.v2.Compressor compressor = 10; + Compressor compressor = 10; +} + +// [#next-free-field: 6] +message Compressor { + // Minimum response length, in bytes, which will trigger compression. The default value is 30. + google.protobuf.UInt32Value content_length = 1; + + // Set of strings that allows specifying which mime-types yield compression; e.g., + // application/json, text/html, etc. When this field is not defined, compression will be applied + // to the following mime-types: "application/javascript", "application/json", + // "application/xhtml+xml", "image/svg+xml", "text/css", "text/html", "text/plain", "text/xml" + // and their synonyms. + repeated string content_type = 2; + + // If true, disables compression when the response contains an etag header. When it is false, the + // filter will preserve weak etags and remove the ones that require strong validation. + bool disable_on_etag_header = 3; + + // If true, removes accept-encoding from the request headers before dispatching it to the upstream + // so that responses do not get compressed before reaching the filter. + // .. attention: + // + // To avoid interfering with other compression filters in the same chain use this option in + // the filter closest to the upstream. + bool remove_accept_encoding_header = 4; + + // Runtime flag that controls whether the filter is enabled or not. If set to false, the + // filter will operate as a pass-through filter. If not specified, defaults to enabled. + config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; } diff --git a/api/envoy/extensions/filters/http/gzip/v4alpha/BUILD b/api/envoy/extensions/filters/http/gzip/v4alpha/BUILD deleted file mode 100644 index 7338b943805e..000000000000 --- a/api/envoy/extensions/filters/http/gzip/v4alpha/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -# DO NOT EDIT. This file is generated by tools/proto_sync.py. - -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = [ - "//envoy/extensions/filters/http/compressor/v3:pkg", - "//envoy/extensions/filters/http/gzip/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - ], -) diff --git a/api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto b/api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto deleted file mode 100644 index 58e7ac78683d..000000000000 --- a/api/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto +++ /dev/null @@ -1,79 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.filters.http.gzip.v4alpha; - -import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; - -import "google/protobuf/wrappers.proto"; - -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.filters.http.gzip.v4alpha"; -option java_outer_classname = "GzipProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; - -// [#protodoc-title: Gzip] -// Gzip :ref:`configuration overview `. -// [#extension: envoy.filters.http.gzip] - -// [#next-free-field: 11] -message Gzip { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.filters.http.gzip.v3.Gzip"; - - enum CompressionStrategy { - DEFAULT = 0; - FILTERED = 1; - HUFFMAN = 2; - RLE = 3; - } - - message CompressionLevel { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.filters.http.gzip.v3.Gzip.CompressionLevel"; - - enum Enum { - DEFAULT = 0; - BEST = 1; - SPEED = 2; - } - } - - reserved 2, 6, 7, 8; - - reserved "content_length", "content_type", "disable_on_etag_header", - "remove_accept_encoding_header"; - - // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values - // use more memory, but are faster and produce better compression results. The default value is 5. - google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; - - // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST" provides higher compression at the cost of - // higher latency, "SPEED" provides lower compression with minimum impact on response time. - // "DEFAULT" provides an optimal result between speed and compression. This field will be set to - // "DEFAULT" if not specified. - CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; - - // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though - // there are situations which changing this parameter might produce better results. For example, - // run-length encoding (RLE) is typically used when the content is known for having sequences - // which same data occurs many consecutive times. For more information about each strategy, please - // refer to zlib manual. - CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; - - // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. - // Larger window results in better compression at the expense of memory usage. The default is 12 - // which will produce a 4096 bytes window. For more details about this parameter, please refer to - // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; - - // Set of configuration parameters common for all compression filters. If this field is set then - // the fields `content_length`, `content_type`, `disable_on_etag_header` and - // `remove_accept_encoding_header` are ignored. - compressor.v3.Compressor compressor = 10; -} diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 42ed4643caba..c6d123acd2c1 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -54,9 +54,6 @@ message Compressor { // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. - // - // This field is ignored when used in the context of the deprecated "envoy.filters.http.gzip" - // filter. CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD index c3e93415440e..6bd8f8337ac8 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/config/filter/http/compressor/v2:pkg", + "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/gzip/v2:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto index 43859be8165c..c51298ac6a20 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.gzip.v3; -import "envoy/config/filter/http/compressor/v2/compressor.proto"; +import "envoy/config/core/v3/base.proto"; import "google/protobuf/wrappers.proto"; @@ -70,7 +70,7 @@ message Gzip { // Set of configuration parameters common for all compression filters. If this field is set then // the fields `content_length`, `content_type`, `disable_on_etag_header` and // `remove_accept_encoding_header` are ignored. - config.filter.http.compressor.v2.Compressor compressor = 10; + Compressor compressor = 10; google.protobuf.UInt32Value hidden_envoy_deprecated_content_length = 2 [deprecated = true]; @@ -80,3 +80,32 @@ message Gzip { bool hidden_envoy_deprecated_remove_accept_encoding_header = 8 [deprecated = true]; } + +// [#next-free-field: 6] +message Compressor { + // Minimum response length, in bytes, which will trigger compression. The default value is 30. + google.protobuf.UInt32Value content_length = 1; + + // Set of strings that allows specifying which mime-types yield compression; e.g., + // application/json, text/html, etc. When this field is not defined, compression will be applied + // to the following mime-types: "application/javascript", "application/json", + // "application/xhtml+xml", "image/svg+xml", "text/css", "text/html", "text/plain", "text/xml" + // and their synonyms. + repeated string content_type = 2; + + // If true, disables compression when the response contains an etag header. When it is false, the + // filter will preserve weak etags and remove the ones that require strong validation. + bool disable_on_etag_header = 3; + + // If true, removes accept-encoding from the request headers before dispatching it to the upstream + // so that responses do not get compressed before reaching the filter. + // .. attention: + // + // To avoid interfering with other compression filters in the same chain use this option in + // the filter closest to the upstream. + bool remove_accept_encoding_header = 4; + + // Runtime flag that controls whether the filter is enabled or not. If set to false, the + // filter will operate as a pass-through filter. If not specified, defaults to enabled. + config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; +} diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD deleted file mode 100644 index 7338b943805e..000000000000 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -# DO NOT EDIT. This file is generated by tools/proto_sync.py. - -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = [ - "//envoy/extensions/filters/http/compressor/v3:pkg", - "//envoy/extensions/filters/http/gzip/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - ], -) diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto deleted file mode 100644 index 58e7ac78683d..000000000000 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v4alpha/gzip.proto +++ /dev/null @@ -1,79 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.filters.http.gzip.v4alpha; - -import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; - -import "google/protobuf/wrappers.proto"; - -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.filters.http.gzip.v4alpha"; -option java_outer_classname = "GzipProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; - -// [#protodoc-title: Gzip] -// Gzip :ref:`configuration overview `. -// [#extension: envoy.filters.http.gzip] - -// [#next-free-field: 11] -message Gzip { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.filters.http.gzip.v3.Gzip"; - - enum CompressionStrategy { - DEFAULT = 0; - FILTERED = 1; - HUFFMAN = 2; - RLE = 3; - } - - message CompressionLevel { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.filters.http.gzip.v3.Gzip.CompressionLevel"; - - enum Enum { - DEFAULT = 0; - BEST = 1; - SPEED = 2; - } - } - - reserved 2, 6, 7, 8; - - reserved "content_length", "content_type", "disable_on_etag_header", - "remove_accept_encoding_header"; - - // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values - // use more memory, but are faster and produce better compression results. The default value is 5. - google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; - - // A value used for selecting the zlib compression level. This setting will affect speed and - // amount of compression applied to the content. "BEST" provides higher compression at the cost of - // higher latency, "SPEED" provides lower compression with minimum impact on response time. - // "DEFAULT" provides an optimal result between speed and compression. This field will be set to - // "DEFAULT" if not specified. - CompressionLevel.Enum compression_level = 3 [(validate.rules).enum = {defined_only: true}]; - - // A value used for selecting the zlib compression strategy which is directly related to the - // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though - // there are situations which changing this parameter might produce better results. For example, - // run-length encoding (RLE) is typically used when the content is known for having sequences - // which same data occurs many consecutive times. For more information about each strategy, please - // refer to zlib manual. - CompressionStrategy compression_strategy = 4 [(validate.rules).enum = {defined_only: true}]; - - // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. - // Larger window results in better compression at the expense of memory usage. The default is 12 - // which will produce a 4096 bytes window. For more details about this parameter, please refer to - // zlib manual > deflateInit2. - google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {lte: 15 gte: 9}]; - - // Set of configuration parameters common for all compression filters. If this field is set then - // the fields `content_length`, `content_type`, `disable_on_etag_header` and - // `remove_accept_encoding_header` are ignored. - compressor.v3.Compressor compressor = 10; -} From 801bdcf7fde0545ab8ff0053548d7d82b290b2e2 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 15 Apr 2020 11:27:11 +0300 Subject: [PATCH 23/56] fix comments Signed-off-by: Dmitry Rozhkov --- api/envoy/extensions/filters/http/gzip/v3/gzip.proto | 3 +++ docs/root/version_history/current.rst | 2 +- .../envoy/extensions/filters/http/gzip/v3/gzip.proto | 3 +++ source/extensions/filters/http/gzip/gzip_filter.cc | 5 +++-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto index 5bdbfcaab521..f134ef3b28d0 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -78,6 +78,9 @@ message Gzip { Compressor compressor = 10; } +// A locally defined Compressor type to use in the gzip filter since the generic Compressor requires +// setting the compressor_library field which doesn't make sense in the context of the deprecated +// gzip filter. // [#next-free-field: 6] message Compressor { // Minimum response length, in bytes, which will trigger compression. The default value is 30. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 2b87dbceb768..5e8641e5a699 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -4,8 +4,8 @@ Changes ------- -* compressor: generic :ref:`compressor ` filter exposed to users. * access loggers: added GRPC_STATUS operator on logging format. +* compressor: generic :ref:`compressor ` filter exposed to users. * http: fixed a bug where the upgrade header was not cleared on responses to non-upgrade requests. Can be reverted temporarily by setting runtime feature `envoy.reloadable_features.fix_upgrade_response` to false. * tracing: tracing configuration has been made fully dynamic and every HTTP connection manager diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto index c51298ac6a20..d21e6c3410d3 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -81,6 +81,9 @@ message Gzip { bool hidden_envoy_deprecated_remove_accept_encoding_header = 8 [deprecated = true]; } +// A locally defined Compressor type to use in the gzip filter since the generic Compressor requires +// setting the compressor_library field which doesn't make sense in the context of the deprecated +// gzip filter. // [#next-free-field: 6] message Compressor { // Minimum response length, in bytes, which will trigger compression. The default value is 30. diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index f3803c914fed..8daaf6fa237a 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -76,8 +76,9 @@ const envoy::extensions::filters::http::compressor::v3::Compressor GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip::v3::Gzip& gzip) { envoy::extensions::filters::http::compressor::v3::Compressor compressor = {}; - // The deprecated gzip filter is using v2 Compressor field to avoid setting the - // compressor_library field which is mandatory in v3. Here we convert v2 Compressor to v3. + // The deprecated gzip filter is using a locally defined Compressor field to avoid setting the + // compressor_library field which is mandatory in v3. Here we convert gzip's Compressor to + // what is accepted by the generic compressor. if (gzip.has_compressor()) { compressor.set_allocated_content_length( new Protobuf::UInt32Value(gzip.compressor().content_length())); From 8e12669f5b7f4cb2daa0eab6582079303c1897d9 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Fri, 17 Apr 2020 12:35:16 +0300 Subject: [PATCH 24/56] restructure compressor library extensions Signed-off-by: Dmitry Rozhkov --- CODEOWNERS | 2 + api/BUILD | 2 +- .../gzip/compressor}/v3/BUILD | 0 .../gzip/compressor}/v3/gzip.proto | 6 +-- .../http/compressor/v3/compressor.proto | 2 +- api/versioning/BUILD | 2 +- .../api-v3/config/compression/compression.rst | 8 +++ docs/root/api-v3/config/config.rst | 1 + docs/root/api-v3/config/filter/http/http.rst | 1 - .../http/http_filters/compressor_filter.rst | 2 +- .../gzip/compressor}/v3/BUILD | 0 .../gzip/compressor}/v3/gzip.proto | 6 +-- include/envoy/compression/compressor/BUILD | 27 ++++++++++ .../envoy/compression/compressor/config.h | 9 ++-- .../envoy/compression/compressor/factory.h | 6 +-- source/extensions/compression/common/BUILD | 19 +++++++ .../common/compressor_factory_base.h} | 23 ++++----- .../gzip/compressor}/BUILD | 4 +- .../gzip/compressor}/config.cc | 49 ++++++++++--------- .../gzip/compressor}/config.h | 37 +++++++------- source/extensions/extensions_build_config.bzl | 7 ++- .../extensions/filters/http/compressor/BUILD | 32 +----------- .../http/compressor/compressor_filter.cc | 2 +- .../http/compressor/compressor_filter.h | 6 +-- .../filters/http/compressor/config.cc | 10 ++-- .../compressor => compression}/gzip/BUILD | 10 ++-- .../gzip/gzip_compressor_library_test.cc | 12 ++--- .../gzip/gzip_filter_test.cc | 7 +-- .../filters/http/common/compressor/BUILD | 2 +- .../compressor/compressor_filter_test.cc | 32 ++++++------ test/extensions/filters/http/compressor/BUILD | 2 +- .../compressor_filter_integration_test.cc | 4 +- 32 files changed, 183 insertions(+), 149 deletions(-) rename api/envoy/extensions/{filters/http/compressor/gzip => compression/gzip/compressor}/v3/BUILD (100%) rename api/envoy/extensions/{filters/http/compressor/gzip => compression/gzip/compressor}/v3/gzip.proto (94%) create mode 100644 docs/root/api-v3/config/compression/compression.rst rename generated_api_shadow/envoy/extensions/{filters/http/compressor/gzip => compression/gzip/compressor}/v3/BUILD (100%) rename generated_api_shadow/envoy/extensions/{filters/http/compressor/gzip => compression/gzip/compressor}/v3/gzip.proto (94%) create mode 100644 include/envoy/compression/compressor/BUILD rename source/extensions/filters/http/compressor/compressor_library_config.h => include/envoy/compression/compressor/config.h (75%) rename source/extensions/filters/http/compressor/compressor_factory.h => include/envoy/compression/compressor/factory.h (82%) create mode 100644 source/extensions/compression/common/BUILD rename source/extensions/{filters/http/compressor/factory_base.h => compression/common/compressor_factory_base.h} (60%) rename source/extensions/{filters/http/compressor/gzip => compression/gzip/compressor}/BUILD (74%) rename source/extensions/{filters/http/compressor/gzip => compression/gzip/compressor}/config.cc (64%) rename source/extensions/{filters/http/compressor/gzip => compression/gzip/compressor}/config.h (56%) rename test/extensions/{filters/http/compressor => compression}/gzip/BUILD (68%) rename test/extensions/{filters/http/compressor => compression}/gzip/gzip_compressor_library_test.cc (96%) rename test/extensions/{filters/http/compressor => compression}/gzip/gzip_filter_test.cc (96%) diff --git a/CODEOWNERS b/CODEOWNERS index 3dd5bfa5b8b5..8eb85aac0728 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -6,6 +6,8 @@ /api/ @envoyproxy/api-shepherds # access loggers /*/extensions/access_loggers/common @auni53 @zuercher +# compression extensions +/*/extensions/compression/ @rojkov @junr03 # csrf extension /*/extensions/filters/http/csrf @dschaller @mattklein123 # original_src http filter extension diff --git a/api/BUILD b/api/BUILD index 9bbbeab31e38..dc1e1c71b49b 100644 --- a/api/BUILD +++ b/api/BUILD @@ -163,6 +163,7 @@ proto_library( "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", + "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/filter/udp/dns_filter/v3alpha:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", "//envoy/extensions/filters/http/adaptive_concurrency/v3:pkg", @@ -170,7 +171,6 @@ proto_library( "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3alpha:pkg", - "//envoy/extensions/filters/http/compressor/gzip/v3:pkg", "//envoy/extensions/filters/http/compressor/v3:pkg", "//envoy/extensions/filters/http/cors/v3:pkg", "//envoy/extensions/filters/http/csrf/v3:pkg", diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD b/api/envoy/extensions/compression/gzip/compressor/v3/BUILD similarity index 100% rename from api/envoy/extensions/filters/http/compressor/gzip/v3/BUILD rename to api/envoy/extensions/compression/gzip/compressor/v3/BUILD diff --git a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/api/envoy/extensions/compression/gzip/compressor/v3/gzip.proto similarity index 94% rename from api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto rename to api/envoy/extensions/compression/gzip/compressor/v3/gzip.proto index cac7115dceba..7508e17df221 100644 --- a/api/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/api/envoy/extensions/compression/gzip/compressor/v3/gzip.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.extensions.filters.http.compressor.gzip.v3; +package envoy.extensions.compression.gzip.compressor.v3; import "google/protobuf/wrappers.proto"; @@ -8,13 +8,13 @@ import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; -option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.gzip.v3"; +option java_package = "io.envoyproxy.envoy.extensions.compression.gzip.compressor.v3"; option java_outer_classname = "GzipProto"; option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] -// [#extension: envoy.filters.http.compressor.gzip] +// [#extension: envoy.compression.gzip.compressor] // [#next-free-field: 6] message Gzip { diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index c6d123acd2c1..7faceb4849bf 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -52,7 +52,7 @@ message Compressor { config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; // A compressor library to use for compression. Currently only - // :ref:`envoy.filters.http.compressor.gzip` + // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } diff --git a/api/versioning/BUILD b/api/versioning/BUILD index cf95d94f2a6b..803b0ab9eca6 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -44,6 +44,7 @@ proto_library( "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", + "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/filter/udp/dns_filter/v3alpha:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", "//envoy/extensions/filters/http/adaptive_concurrency/v3:pkg", @@ -51,7 +52,6 @@ proto_library( "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3alpha:pkg", - "//envoy/extensions/filters/http/compressor/gzip/v3:pkg", "//envoy/extensions/filters/http/compressor/v3:pkg", "//envoy/extensions/filters/http/cors/v3:pkg", "//envoy/extensions/filters/http/csrf/v3:pkg", diff --git a/docs/root/api-v3/config/compression/compression.rst b/docs/root/api-v3/config/compression/compression.rst new file mode 100644 index 000000000000..80aa0ba927cc --- /dev/null +++ b/docs/root/api-v3/config/compression/compression.rst @@ -0,0 +1,8 @@ +Compression +=========== + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../extensions/compression/gzip/*/v3/* diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst index 3725a558c7c8..ddcd33416ca3 100644 --- a/docs/root/api-v3/config/config.rst +++ b/docs/root/api-v3/config/config.rst @@ -12,6 +12,7 @@ Extensions transport_socket/transport_socket resource_monitor/resource_monitor common/common + compression/compression cluster/cluster grpc_credential/grpc_credential retry/retry diff --git a/docs/root/api-v3/config/filter/http/http.rst b/docs/root/api-v3/config/filter/http/http.rst index a5090c0634c6..20f2c75664db 100644 --- a/docs/root/api-v3/config/filter/http/http.rst +++ b/docs/root/api-v3/config/filter/http/http.rst @@ -7,4 +7,3 @@ HTTP filters */empty/* ../../../extensions/filters/http/*/v3/* - ../../../extensions/filters/http/compressor/*/v3/* diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst index 26c70b84aa53..5bf7ef806ecb 100644 --- a/docs/root/configuration/http/http_filters/compressor_filter.rst +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -23,7 +23,7 @@ determine whether or not the content should be compressed. The content is compressed and then sent to the client with the appropriate headers, if response and request allow. -Currently the filter supports :ref:`gzip compression ` +Currently the filter supports :ref:`gzip compression ` only. Other compression libraries can be supported as extensions. An example configuration of the filter may look like the following: diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD b/generated_api_shadow/envoy/extensions/compression/gzip/compressor/v3/BUILD similarity index 100% rename from generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/BUILD rename to generated_api_shadow/envoy/extensions/compression/gzip/compressor/v3/BUILD diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/compression/gzip/compressor/v3/gzip.proto similarity index 94% rename from generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto rename to generated_api_shadow/envoy/extensions/compression/gzip/compressor/v3/gzip.proto index cac7115dceba..7508e17df221 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/compression/gzip/compressor/v3/gzip.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.extensions.filters.http.compressor.gzip.v3; +package envoy.extensions.compression.gzip.compressor.v3; import "google/protobuf/wrappers.proto"; @@ -8,13 +8,13 @@ import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; -option java_package = "io.envoyproxy.envoy.extensions.filters.http.compressor.gzip.v3"; +option java_package = "io.envoyproxy.envoy.extensions.compression.gzip.compressor.v3"; option java_outer_classname = "GzipProto"; option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Gzip] -// [#extension: envoy.filters.http.compressor.gzip] +// [#extension: envoy.compression.gzip.compressor] // [#next-free-field: 6] message Gzip { diff --git a/include/envoy/compression/compressor/BUILD b/include/envoy/compression/compressor/BUILD new file mode 100644 index 000000000000..3e220abff580 --- /dev/null +++ b/include/envoy/compression/compressor/BUILD @@ -0,0 +1,27 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "compressor_factory_interface", + hdrs = ["factory.h"], + deps = [ + "//include/envoy/compressor:compressor_interface", + ], +) + +envoy_cc_library( + name = "compressor_library_config_interface", + hdrs = ["config.h"], + deps = [ + ":compressor_factory_interface", + "//include/envoy/config:typed_config_interface", + "//include/envoy/server:filter_config_interface", + ], +) diff --git a/source/extensions/filters/http/compressor/compressor_library_config.h b/include/envoy/compression/compressor/config.h similarity index 75% rename from source/extensions/filters/http/compressor/compressor_library_config.h rename to include/envoy/compression/compressor/config.h index fb6d7fa822d4..af8f0b9997fa 100644 --- a/source/extensions/filters/http/compressor/compressor_library_config.h +++ b/include/envoy/compression/compressor/config.h @@ -1,13 +1,11 @@ #pragma once +#include "envoy/compression/compressor/factory.h" #include "envoy/config/typed_config.h" #include "envoy/server/filter_config.h" -#include "extensions/filters/http/compressor/compressor_factory.h" - namespace Envoy { -namespace Extensions { -namespace HttpFilters { +namespace Compression { namespace Compressor { class NamedCompressorLibraryConfigFactory : public Config::TypedFactory { @@ -20,6 +18,5 @@ class NamedCompressorLibraryConfigFactory : public Config::TypedFactory { }; } // namespace Compressor -} // namespace HttpFilters -} // namespace Extensions +} // namespace Compression } // namespace Envoy diff --git a/source/extensions/filters/http/compressor/compressor_factory.h b/include/envoy/compression/compressor/factory.h similarity index 82% rename from source/extensions/filters/http/compressor/compressor_factory.h rename to include/envoy/compression/compressor/factory.h index 229594999830..a5f9996bc61f 100644 --- a/source/extensions/filters/http/compressor/compressor_factory.h +++ b/include/envoy/compression/compressor/factory.h @@ -3,8 +3,7 @@ #include "envoy/compressor/compressor.h" namespace Envoy { -namespace Extensions { -namespace HttpFilters { +namespace Compression { namespace Compressor { class CompressorFactory { @@ -19,6 +18,5 @@ class CompressorFactory { using CompressorFactoryPtr = std::unique_ptr; } // namespace Compressor -} // namespace HttpFilters -} // namespace Extensions +} // namespace Compression } // namespace Envoy diff --git a/source/extensions/compression/common/BUILD b/source/extensions/compression/common/BUILD new file mode 100644 index 000000000000..ad0fe31a359f --- /dev/null +++ b/source/extensions/compression/common/BUILD @@ -0,0 +1,19 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "compressor_factory_base_lib", + hdrs = ["compressor_factory_base.h"], + deps = [ + "//include/envoy/compression/compressor:compressor_factory_interface", + "//include/envoy/compression/compressor:compressor_library_config_interface", + "//include/envoy/server:filter_config_interface", + ], +) diff --git a/source/extensions/filters/http/compressor/factory_base.h b/source/extensions/compression/common/compressor_factory_base.h similarity index 60% rename from source/extensions/filters/http/compressor/factory_base.h rename to source/extensions/compression/common/compressor_factory_base.h index cc3ec0e52925..0f6095f5e151 100644 --- a/source/extensions/filters/http/compressor/factory_base.h +++ b/source/extensions/compression/common/compressor_factory_base.h @@ -1,19 +1,19 @@ #pragma once +#include "envoy/compression/compressor/config.h" +#include "envoy/compression/compressor/factory.h" #include "envoy/server/filter_config.h" -#include "extensions/filters/http/compressor/compressor_factory.h" -#include "extensions/filters/http/compressor/compressor_library_config.h" - namespace Envoy { namespace Extensions { -namespace HttpFilters { -namespace Compressor { +namespace Compression { +namespace Common { template -class CompressorLibraryFactoryBase : public NamedCompressorLibraryConfigFactory { +class CompressorLibraryFactoryBase + : public Envoy::Compression::Compressor::NamedCompressorLibraryConfigFactory { public: - CompressorFactoryPtr + Envoy::Compression::Compressor::CompressorFactoryPtr createCompressorFactoryFromProto(const Protobuf::Message& proto_config, Server::Configuration::FactoryContext& context) override { return createCompressorFactoryFromProtoTyped( @@ -25,7 +25,7 @@ class CompressorLibraryFactoryBase : public NamedCompressorLibraryConfigFactory return std::make_unique(); } - std::string category() const override { return "envoy.filters.http.compressor"; } + std::string category() const override { return "envoy.compression.compressor"; } std::string name() const override { return name_; } @@ -33,11 +33,12 @@ class CompressorLibraryFactoryBase : public NamedCompressorLibraryConfigFactory CompressorLibraryFactoryBase(const std::string& name) : name_(name) {} private: - virtual CompressorFactoryPtr createCompressorFactoryFromProtoTyped(const ConfigProto&) PURE; + virtual Envoy::Compression::Compressor::CompressorFactoryPtr + createCompressorFactoryFromProtoTyped(const ConfigProto&) PURE; const std::string name_; }; -} // namespace Compressor -} // namespace HttpFilters +} // namespace Common +} // namespace Compression } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/filters/http/compressor/gzip/BUILD b/source/extensions/compression/gzip/compressor/BUILD similarity index 74% rename from source/extensions/filters/http/compressor/gzip/BUILD rename to source/extensions/compression/gzip/compressor/BUILD index 2c22ff68d107..5c8a3276d5d4 100644 --- a/source/extensions/filters/http/compressor/gzip/BUILD +++ b/source/extensions/compression/gzip/compressor/BUILD @@ -16,8 +16,8 @@ envoy_cc_extension( deps = [ "//source/common/compressor:compressor_lib", "//source/common/http:headers_lib", + "//source/extensions/compression/common:compressor_factory_base_lib", "//source/extensions/filters/http:well_known_names", - "//source/extensions/filters/http/compressor:factory_base_lib", - "@envoy_api//envoy/extensions/filters/http/compressor/gzip/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/compression/gzip/compressor/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/compressor/gzip/config.cc b/source/extensions/compression/gzip/compressor/config.cc similarity index 64% rename from source/extensions/filters/http/compressor/gzip/config.cc rename to source/extensions/compression/gzip/compressor/config.cc index 4336f5578f96..fd529b3b5057 100644 --- a/source/extensions/filters/http/compressor/gzip/config.cc +++ b/source/extensions/compression/gzip/compressor/config.cc @@ -1,10 +1,10 @@ -#include "extensions/filters/http/compressor/gzip/config.h" +#include "extensions/compression/gzip/compressor/config.h" namespace Envoy { namespace Extensions { -namespace HttpFilters { -namespace Compressor { +namespace Compression { namespace Gzip { +namespace Compressor { namespace { // Default zlib memory level. @@ -21,7 +21,7 @@ const uint32_t DefaultChunkSize = 4096; } // namespace GzipCompressorFactory::GzipCompressorFactory( - const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& gzip) + const envoy::extensions::compression::gzip::compressor::v3::Gzip& gzip) : compression_level_(compressionLevelEnum(gzip.compression_level())), compression_strategy_(compressionStrategyEnum(gzip.compression_strategy())), memory_level_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(gzip, memory_level, DefaultMemoryLevel)), @@ -30,26 +30,26 @@ GzipCompressorFactory::GzipCompressorFactory( chunk_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(gzip, chunk_size, DefaultChunkSize)) {} Envoy::Compressor::ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::compressionLevelEnum( - envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel + envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionLevel compression_level) { switch (compression_level) { - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::BEST_SPEED: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::BEST_SPEED: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Speed; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_2: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_2: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level2; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_3: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_3: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level3; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_4: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_4: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level4; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_5: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_5: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level5; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_6: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_6: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level6; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_7: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_7: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level7; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::COMPRESSION_LEVEL_8: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_8: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level8; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::BEST_COMPRESSION: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::BEST_COMPRESSION: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best; default: return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard; @@ -58,16 +58,16 @@ Envoy::Compressor::ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::c Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy GzipCompressorFactory::compressionStrategyEnum( - envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionStrategy + envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionStrategy compression_strategy) { switch (compression_strategy) { - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::FILTERED: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::FILTERED: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::FIXED: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::FIXED: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::HUFFMAN_ONLY: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::HUFFMAN_ONLY: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman; - case envoy::extensions::filters::http::compressor::gzip::v3::Gzip::RLE: + case envoy::extensions::compression::gzip::compressor::v3::Gzip::RLE: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle; default: return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard; @@ -80,19 +80,20 @@ Envoy::Compressor::CompressorPtr GzipCompressorFactory::createCompressor() { return compressor; } -Compressor::CompressorFactoryPtr +Envoy::Compression::Compressor::CompressorFactoryPtr GzipCompressorLibraryFactory::createCompressorFactoryFromProtoTyped( - const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& proto_config) { + const envoy::extensions::compression::gzip::compressor::v3::Gzip& proto_config) { return std::make_unique(proto_config); } /** * Static registration for the gzip compressor library. @see NamedCompressorLibraryConfigFactory. */ -REGISTER_FACTORY(GzipCompressorLibraryFactory, NamedCompressorLibraryConfigFactory); +REGISTER_FACTORY(GzipCompressorLibraryFactory, + Envoy::Compression::Compressor::NamedCompressorLibraryConfigFactory); -} // namespace Gzip } // namespace Compressor -} // namespace HttpFilters +} // namespace Gzip +} // namespace Compression } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/filters/http/compressor/gzip/config.h b/source/extensions/compression/gzip/compressor/config.h similarity index 56% rename from source/extensions/filters/http/compressor/gzip/config.h rename to source/extensions/compression/gzip/compressor/config.h index 798181f91a12..024eb9a6c93c 100644 --- a/source/extensions/filters/http/compressor/gzip/config.h +++ b/source/extensions/compression/gzip/compressor/config.h @@ -1,32 +1,33 @@ #pragma once -#include "envoy/extensions/filters/http/compressor/gzip/v3/gzip.pb.h" -#include "envoy/extensions/filters/http/compressor/gzip/v3/gzip.pb.validate.h" +#include "envoy/compression/compressor/factory.h" +#include "envoy/extensions/compression/gzip/compressor/v3/gzip.pb.h" +#include "envoy/extensions/compression/gzip/compressor/v3/gzip.pb.validate.h" #include "common/compressor/zlib_compressor_impl.h" #include "common/http/headers.h" -#include "extensions/filters/http/compressor/factory_base.h" +#include "extensions/compression/common/compressor_factory_base.h" #include "extensions/filters/http/well_known_names.h" namespace Envoy { namespace Extensions { -namespace HttpFilters { -namespace Compressor { +namespace Compression { namespace Gzip { +namespace Compressor { namespace { const std::string& gzipStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "gzip."); } const std::string& gzipExtensionName() { - CONSTRUCT_ON_FIRST_USE(std::string, "envoy.filters.http.compressor.gzip"); + CONSTRUCT_ON_FIRST_USE(std::string, "envoy.compression.gzip.compressor"); } } // namespace -class GzipCompressorFactory : public Compressor::CompressorFactory { +class GzipCompressorFactory : public Envoy::Compression::Compressor::CompressorFactory { public: - GzipCompressorFactory(const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& gzip); + GzipCompressorFactory(const envoy::extensions::compression::gzip::compressor::v3::Gzip& gzip); Envoy::Compressor::CompressorPtr createCompressor() override; const std::string& statsPrefix() const override { return gzipStatsPrefix(); } @@ -37,11 +38,11 @@ class GzipCompressorFactory : public Compressor::CompressorFactory { private: friend class GzipTest; - static Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compressionLevelEnum( - envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionLevel - compression_level); + static Envoy::Compressor::ZlibCompressorImpl::CompressionLevel + compressionLevelEnum(envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionLevel + compression_level); static Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategyEnum( - envoy::extensions::filters::http::compressor::gzip::v3::Gzip::CompressionStrategy + envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionStrategy compression_strategy); Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compression_level_; @@ -53,20 +54,20 @@ class GzipCompressorFactory : public Compressor::CompressorFactory { }; class GzipCompressorLibraryFactory - : public CompressorLibraryFactoryBase< - envoy::extensions::filters::http::compressor::gzip::v3::Gzip> { + : public Common::CompressorLibraryFactoryBase< + envoy::extensions::compression::gzip::compressor::v3::Gzip> { public: GzipCompressorLibraryFactory() : CompressorLibraryFactoryBase(gzipExtensionName()) {} private: - Compressor::CompressorFactoryPtr createCompressorFactoryFromProtoTyped( - const envoy::extensions::filters::http::compressor::gzip::v3::Gzip& config) override; + Envoy::Compression::Compressor::CompressorFactoryPtr createCompressorFactoryFromProtoTyped( + const envoy::extensions::compression::gzip::compressor::v3::Gzip& config) override; }; DECLARE_FACTORY(GzipCompressorLibraryFactory); -} // namespace Gzip } // namespace Compressor -} // namespace HttpFilters +} // namespace Gzip +} // namespace Compression } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 57e7b813d94f..e6705473ba81 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -16,6 +16,12 @@ EXTENSIONS = { "envoy.clusters.dynamic_forward_proxy": "//source/extensions/clusters/dynamic_forward_proxy:cluster", "envoy.clusters.redis": "//source/extensions/clusters/redis:redis_cluster", + # + # Compression + # + + "envoy.compression.gzip.compressor": "//source/extensions/compression/gzip/compressor:config", + # # gRPC Credentials Plugins # @@ -39,7 +45,6 @@ EXTENSIONS = { "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cache": "//source/extensions/filters/http/cache:config", "envoy.filters.http.compressor": "//source/extensions/filters/http/compressor:config", - "envoy.filters.http.compressor.gzip": "//source/extensions/filters/http/compressor/gzip:config", "envoy.filters.http.cors": "//source/extensions/filters/http/cors:config", "envoy.filters.http.csrf": "//source/extensions/filters/http/csrf:config", "envoy.filters.http.dynamic_forward_proxy": "//source/extensions/filters/http/dynamic_forward_proxy:config", diff --git a/source/extensions/filters/http/compressor/BUILD b/source/extensions/filters/http/compressor/BUILD index 2b79be2a8b22..dae412be9ca0 100644 --- a/source/extensions/filters/http/compressor/BUILD +++ b/source/extensions/filters/http/compressor/BUILD @@ -12,45 +12,17 @@ load( envoy_package() -envoy_cc_library( - name = "compressor_factory_interface", - hdrs = ["compressor_factory.h"], - deps = [ - "//include/envoy/compressor:compressor_interface", - ], -) - envoy_cc_library( name = "compressor_filter_lib", srcs = ["compressor_filter.cc"], hdrs = ["compressor_filter.h"], deps = [ - ":compressor_factory_interface", + "//include/envoy/compression/compressor:compressor_factory_interface", "//source/extensions/filters/http/common/compressor:compressor_lib", "@envoy_api//envoy/extensions/filters/http/compressor/v3:pkg_cc_proto", ], ) -envoy_cc_library( - name = "compressor_library_config_interface", - hdrs = ["compressor_library_config.h"], - deps = [ - ":compressor_factory_interface", - "//include/envoy/config:typed_config_interface", - "//include/envoy/server:filter_config_interface", - ], -) - -envoy_cc_library( - name = "factory_base_lib", - hdrs = ["factory_base.h"], - deps = [ - ":compressor_factory_interface", - ":compressor_library_config_interface", - "//include/envoy/server:filter_config_interface", - ], -) - envoy_cc_extension( name = "config", srcs = ["config.cc"], @@ -58,7 +30,7 @@ envoy_cc_extension( security_posture = "robust_to_untrusted_downstream", deps = [ ":compressor_filter_lib", - ":compressor_library_config_interface", + "//include/envoy/compression/compressor:compressor_library_config_interface", "//source/common/config:utility_lib", "//source/extensions/filters/http:well_known_names", "//source/extensions/filters/http/common:factory_base_lib", diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index e70f942ea911..057f8af67e45 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -8,7 +8,7 @@ namespace Compressor { CompressorFilterConfig::CompressorFilterConfig( const envoy::extensions::filters::http::compressor::v3::Compressor& generic_compressor, const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, - CompressorFactoryPtr compressor_factory) + Compression::Compressor::CompressorFactoryPtr compressor_factory) : Common::Compressors::CompressorFilterConfig( generic_compressor, stats_prefix + compressor_factory->statsPrefix(), scope, runtime, compressor_factory->contentEncoding()), diff --git a/source/extensions/filters/http/compressor/compressor_filter.h b/source/extensions/filters/http/compressor/compressor_filter.h index 74c82aa7071c..9e648fece872 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.h +++ b/source/extensions/filters/http/compressor/compressor_filter.h @@ -1,9 +1,9 @@ #pragma once +#include "envoy/compression/compressor/factory.h" #include "envoy/extensions/filters/http/compressor/v3/compressor.pb.h" #include "extensions/filters/http/common/compressor/compressor.h" -#include "extensions/filters/http/compressor/compressor_factory.h" namespace Envoy { namespace Extensions { @@ -21,12 +21,12 @@ class CompressorFilterConfig : public Common::Compressors::CompressorFilterConfi CompressorFilterConfig( const envoy::extensions::filters::http::compressor::v3::Compressor& genereic_compressor, const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, - CompressorFactoryPtr compressor_factory); + Envoy::Compression::Compressor::CompressorFactoryPtr compressor_factory); Envoy::Compressor::CompressorPtr makeCompressor() override; private: - const CompressorFactoryPtr compressor_factory_; + const Envoy::Compression::Compressor::CompressorFactoryPtr compressor_factory_; }; } // namespace Compressor diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc index 1e7608e40412..1fb2c7f55c61 100644 --- a/source/extensions/filters/http/compressor/config.cc +++ b/source/extensions/filters/http/compressor/config.cc @@ -1,9 +1,10 @@ #include "extensions/filters/http/compressor/config.h" +#include "envoy/compression/compressor/config.h" + #include "common/config/utility.h" #include "extensions/filters/http/compressor/compressor_filter.h" -#include "extensions/filters/http/compressor/compressor_library_config.h" namespace Envoy { namespace Extensions { @@ -15,8 +16,9 @@ Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { const std::string type{TypeUtil::typeUrlToDescriptorFullName( proto_config.compressor_library().typed_config().type_url())}; - NamedCompressorLibraryConfigFactory* const config_factory = - Registry::FactoryRegistry::getFactoryByType(type); + Compression::Compressor::NamedCompressorLibraryConfigFactory* const config_factory = + Registry::FactoryRegistry< + Compression::Compressor::NamedCompressorLibraryConfigFactory>::getFactoryByType(type); if (config_factory == nullptr) { throw EnvoyException( fmt::format("Didn't find a registered implementation for type: '{}'", type)); @@ -24,7 +26,7 @@ Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig( proto_config.compressor_library().typed_config(), context.messageValidationVisitor(), *config_factory); - CompressorFactoryPtr compressor_factory = + Compression::Compressor::CompressorFactoryPtr compressor_factory = config_factory->createCompressorFactoryFromProto(*message, context); Common::Compressors::CompressorFilterConfigSharedPtr config = std::make_shared(proto_config, stats_prefix, context.scope(), diff --git a/test/extensions/filters/http/compressor/gzip/BUILD b/test/extensions/compression/gzip/BUILD similarity index 68% rename from test/extensions/filters/http/compressor/gzip/BUILD rename to test/extensions/compression/gzip/BUILD index c666743a44b9..c36c79504e58 100644 --- a/test/extensions/filters/http/compressor/gzip/BUILD +++ b/test/extensions/compression/gzip/BUILD @@ -14,21 +14,21 @@ envoy_package() envoy_extension_cc_test( name = "gzip_compressor_library_test", srcs = ["gzip_compressor_library_test.cc"], - extension_name = "envoy.filters.http.compressor.gzip", + extension_name = "envoy.compression.gzip.compressor", deps = [ - "//source/extensions/filters/http/compressor/gzip:config", + "//source/extensions/compression/gzip/compressor:config", "//test/test_common:utility_lib", ], ) envoy_extension_cc_test( - name = "gzip_filter_test", + name = "gzip_compressor_integration_test", srcs = ["gzip_filter_test.cc"], - extension_name = "envoy.filters.http.compressor.gzip", + extension_name = "envoy.compression.gzip.compressor", deps = [ "//source/common/decompressor:decompressor_lib", + "//source/extensions/compression/gzip/compressor:config", "//source/extensions/filters/http/compressor:compressor_filter_lib", - "//source/extensions/filters/http/compressor/gzip:config", "//test/mocks/http:http_mocks", ], ) diff --git a/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc b/test/extensions/compression/gzip/gzip_compressor_library_test.cc similarity index 96% rename from test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc rename to test/extensions/compression/gzip/gzip_compressor_library_test.cc index 90b7405f354d..2e339bd7adfb 100644 --- a/test/extensions/filters/http/compressor/gzip/gzip_compressor_library_test.cc +++ b/test/extensions/compression/gzip/gzip_compressor_library_test.cc @@ -1,4 +1,4 @@ -#include "extensions/filters/http/compressor/gzip/config.h" +#include "extensions/compression/gzip/compressor/config.h" #include "test/test_common/utility.h" @@ -6,9 +6,9 @@ namespace Envoy { namespace Extensions { -namespace HttpFilters { -namespace Compressor { +namespace Compression { namespace Gzip { +namespace Compressor { class GzipTest : public testing::Test { protected: @@ -16,7 +16,7 @@ class GzipTest : public testing::Test { // GzipTest Helpers void setUpGzip(std::string&& json) { - envoy::extensions::filters::http::compressor::gzip::v3::Gzip gzip; + envoy::extensions::compression::gzip::compressor::v3::Gzip gzip; TestUtility::loadFromJson(json, gzip); factory_ = std::make_unique(gzip); } @@ -102,8 +102,8 @@ TEST_F(GzipTest, AvailableCombinationCompressionStrategyAndLevelConfig) { Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level9, "COMPRESSION_LEVEL_9"); } -} // namespace Gzip } // namespace Compressor -} // namespace HttpFilters +} // namespace Gzip +} // namespace Compression } // namespace Extensions } // namespace Envoy diff --git a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc b/test/extensions/compression/gzip/gzip_filter_test.cc similarity index 96% rename from test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc rename to test/extensions/compression/gzip/gzip_filter_test.cc index 6b77e67d5b11..d3416896af07 100644 --- a/test/extensions/filters/http/compressor/gzip/gzip_filter_test.cc +++ b/test/extensions/compression/gzip/gzip_filter_test.cc @@ -3,8 +3,8 @@ #include "common/common/hex.h" #include "common/decompressor/zlib_decompressor_impl.h" +#include "extensions/compression/gzip/compressor/config.h" #include "extensions/filters/http/compressor/compressor_filter.h" -#include "extensions/filters/http/compressor/gzip/config.h" #include "test/mocks/http/mocks.h" #include "test/mocks/stats/mocks.h" @@ -18,8 +18,9 @@ class GzipCompressorFilterTest : public testing::Test { protected: void SetUp() override { envoy::extensions::filters::http::compressor::v3::Compressor compressor; - envoy::extensions::filters::http::compressor::gzip::v3::Gzip gzip; - CompressorFactoryPtr gzip_factory = std::make_unique(gzip); + envoy::extensions::compression::gzip::compressor::v3::Gzip gzip; + Envoy::Compression::Compressor::CompressorFactoryPtr gzip_factory = + std::make_unique(gzip); config_ = std::make_shared(compressor, "test.", stats_, runtime_, std::move(gzip_factory)); filter_ = std::make_unique(config_); diff --git a/test/extensions/filters/http/common/compressor/BUILD b/test/extensions/filters/http/common/compressor/BUILD index e08df739e012..4d79ef96c075 100644 --- a/test/extensions/filters/http/common/compressor/BUILD +++ b/test/extensions/filters/http/common/compressor/BUILD @@ -13,8 +13,8 @@ envoy_cc_test( srcs = ["compressor_filter_test.cc"], deps = [ "//source/common/protobuf:utility_lib", + "//source/extensions/compression/gzip/compressor:config", "//source/extensions/filters/http/common/compressor:compressor_lib", - "//source/extensions/filters/http/compressor/gzip:config", "//test/mocks/http:http_mocks", "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index 60e9346d5fdd..a55ad21ee108 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -50,7 +50,7 @@ class CompressorFilterTest : public testing::Test { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -185,7 +185,7 @@ TEST_F(CompressorFilterTest, DecodeHeadersWithRuntimeDisabled) { }, "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -368,7 +368,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -401,7 +401,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -430,7 +430,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -459,7 +459,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -488,7 +488,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -527,7 +527,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -589,7 +589,7 @@ TEST_F(CompressorFilterTest, IsMinimumContentLength) { "content_length": 500, "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -621,7 +621,7 @@ TEST_F(CompressorFilterTest, ContentLengthCompression) { "content_length": 500, "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -691,7 +691,7 @@ TEST_F(CompressorFilterTest, IsContentTypeAllowed) { ], "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -734,7 +734,7 @@ TEST_F(CompressorFilterTest, ContentTypeNoCompression) { ], "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -798,7 +798,7 @@ TEST_F(CompressorFilterTest, IsEtagAllowed) { "disable_on_etag_header": true, "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -827,7 +827,7 @@ TEST_F(CompressorFilterTest, EtagNoCompression) { "disable_on_etag_header": true, "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -1004,7 +1004,7 @@ TEST_F(CompressorFilterTest, RemoveAcceptEncodingHeader) { "remove_accept_encoding_header": true, "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } @@ -1018,7 +1018,7 @@ TEST_F(CompressorFilterTest, RemoveAcceptEncodingHeader) { { "compressor_library": { "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip" + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" } } } diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD index 7e84ab2a8927..d192a617454f 100644 --- a/test/extensions/filters/http/compressor/BUILD +++ b/test/extensions/filters/http/compressor/BUILD @@ -19,8 +19,8 @@ envoy_extension_cc_test( extension_name = "envoy.filters.http.compressor", deps = [ "//source/common/decompressor:decompressor_lib", + "//source/extensions/compression/gzip/compressor:config", "//source/extensions/filters/http/compressor:config", - "//source/extensions/filters/http/compressor/gzip:config", "//test/integration:http_integration_lib", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index b1c9014c2fbb..92c4ca5b3657 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -76,7 +76,7 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Fri, 17 Apr 2020 18:04:30 +0300 Subject: [PATCH 25/56] run proto_format/proto_format.sh to fix format check Signed-off-by: Dmitry Rozhkov --- .../extensions/filters/http/compressor/v3/compressor.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index c6d123acd2c1..7faceb4849bf 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -52,7 +52,7 @@ message Compressor { config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; // A compressor library to use for compression. Currently only - // :ref:`envoy.filters.http.compressor.gzip` + // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; } From d72e01586aadd32652b8e2717b54891c2b878ad6 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 20 Apr 2020 11:03:17 +0300 Subject: [PATCH 26/56] finalize moving gzip code to new compression extension Signed-off-by: Dmitry Rozhkov --- api/BUILD | 1 + .../compression/common/compressor/v3/BUILD | 9 ++ .../common/compressor/v3/compressor.proto | 22 ++++ .../filters/http/compressor/v3/BUILD | 1 + .../http/compressor/v3/compressor.proto | 10 +- api/versioning/BUILD | 1 + .../api-v3/config/compression/compression.rst | 1 + .../compression/common/compressor/v3/BUILD | 9 ++ .../common/compressor/v3/compressor.proto | 22 ++++ .../filters/http/compressor/v3/BUILD | 1 + .../http/compressor/v3/compressor.proto | 10 +- include/envoy/compression/compressor/BUILD | 10 +- .../{ => compression}/compressor/compressor.h | 2 + .../envoy/compression/compressor/factory.h | 4 +- .../{ => compression}/decompressor/BUILD | 0 .../decompressor/decompressor.h | 2 + source/common/common/BUILD | 10 -- source/common/compressor/BUILD | 22 ---- .../compression/common/{ => compressor}/BUILD | 2 +- .../factory_base.h} | 2 + .../extensions/compression/gzip/common}/BUILD | 8 +- .../compression/gzip/common}/base.cc | 12 +- .../compression/gzip/common}/base.h | 10 +- .../compression/gzip/compressor/BUILD | 18 ++- .../compression/gzip/compressor/config.cc | 39 +++---- .../compression/gzip/compressor/config.h | 16 +-- .../gzip}/compressor/zlib_compressor_impl.cc | 15 ++- .../gzip}/compressor/zlib_compressor_impl.h | 14 ++- .../compression/gzip}/decompressor/BUILD | 4 +- .../decompressor/zlib_decompressor_impl.cc | 10 +- .../decompressor/zlib_decompressor_impl.h | 15 ++- .../filters/http/common/compressor/BUILD | 2 +- .../http/common/compressor/compressor.cc | 5 +- .../http/common/compressor/compressor.h | 6 +- .../http/compressor/compressor_filter.cc | 2 +- .../http/compressor/compressor_filter.h | 2 +- source/extensions/filters/http/gzip/BUILD | 2 +- .../filters/http/gzip/gzip_filter.cc | 24 ++-- .../filters/http/gzip/gzip_filter.h | 19 +-- test/common/stats/BUILD | 2 +- test/extensions/compression/gzip/BUILD | 34 ------ .../compression/gzip}/compressor/BUILD | 20 +++- ...ized-compressor_fuzz_test-5149986500640768 | Bin ...ized-compressor_fuzz_test-5407695477932032 | Bin ...ized-compressor_fuzz_test-5644831560302592 | Bin ...ized-compressor_fuzz_test-6005942746873856 | Bin .../gzip}/compressor/compressor_corpus/empty | 0 .../gzip}/compressor/compressor_corpus/noise | Bin .../gzip}/compressor/compressor_corpus/simple | 0 .../gzip}/compressor/compressor_fuzz_test.cc | 14 ++- .../gzip_compressor_library_test.cc | 107 +++++++++++++++++ .../compressor/zlib_compressor_impl_test.cc | 29 +++-- .../compression/gzip}/decompressor/BUILD | 4 +- .../zlib_decompressor_impl_test.cc | 69 ++++++----- .../gzip/gzip_compressor_library_test.cc | 109 ------------------ .../compressor/compressor_filter_test.cc | 8 +- test/extensions/filters/http/compressor/BUILD | 14 ++- .../compressor_filter_integration_test.cc | 4 +- .../gzip_compressor_filter_test.cc} | 4 +- test/extensions/filters/http/gzip/BUILD | 6 +- .../http/gzip/gzip_filter_integration_test.cc | 4 +- .../filters/http/gzip/gzip_filter_test.cc | 32 ++--- 62 files changed, 471 insertions(+), 353 deletions(-) create mode 100644 api/envoy/extensions/compression/common/compressor/v3/BUILD create mode 100644 api/envoy/extensions/compression/common/compressor/v3/compressor.proto create mode 100644 generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD create mode 100644 generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto rename include/envoy/{ => compression}/compressor/compressor.h (93%) rename include/envoy/{ => compression}/decompressor/BUILD (100%) rename include/envoy/{ => compression}/decompressor/decompressor.h (92%) delete mode 100644 source/common/compressor/BUILD rename source/extensions/compression/common/{ => compressor}/BUILD (91%) rename source/extensions/compression/common/{compressor_factory_base.h => compressor/factory_base.h} (96%) rename {include/envoy/compressor => source/extensions/compression/gzip/common}/BUILD (54%) rename source/{common/common/zlib => extensions/compression/gzip/common}/base.cc (73%) rename source/{common/common/zlib => extensions/compression/gzip/common}/base.h (84%) rename source/{common => extensions/compression/gzip}/compressor/zlib_compressor_impl.cc (83%) rename source/{common => extensions/compression/gzip}/compressor/zlib_compressor_impl.h (87%) rename source/{common => extensions/compression/gzip}/decompressor/BUILD (76%) rename source/{common => extensions/compression/gzip}/decompressor/zlib_decompressor_impl.cc (89%) rename source/{common => extensions/compression/gzip}/decompressor/zlib_decompressor_impl.h (81%) delete mode 100644 test/extensions/compression/gzip/BUILD rename test/{common => extensions/compression/gzip}/compressor/BUILD (51%) rename test/{common => extensions/compression/gzip}/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 (100%) rename test/{common => extensions/compression/gzip}/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 (100%) rename test/{common => extensions/compression/gzip}/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 (100%) rename test/{common => extensions/compression/gzip}/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 (100%) rename test/{common => extensions/compression/gzip}/compressor/compressor_corpus/empty (100%) rename test/{common => extensions/compression/gzip}/compressor/compressor_corpus/noise (100%) rename test/{common => extensions/compression/gzip}/compressor/compressor_corpus/simple (100%) rename test/{common => extensions/compression/gzip}/compressor/compressor_fuzz_test.cc (86%) create mode 100644 test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc rename test/{common => extensions/compression/gzip}/compressor/zlib_compressor_impl_test.cc (89%) rename test/{common => extensions/compression/gzip}/decompressor/BUILD (71%) rename test/{common => extensions/compression/gzip}/decompressor/zlib_decompressor_impl_test.cc (78%) delete mode 100644 test/extensions/compression/gzip/gzip_compressor_library_test.cc rename test/extensions/{compression/gzip/gzip_filter_test.cc => filters/http/compressor/gzip_compressor_filter_test.cc} (98%) diff --git a/api/BUILD b/api/BUILD index 9385e79f0fd1..444c3d85a6c4 100644 --- a/api/BUILD +++ b/api/BUILD @@ -163,6 +163,7 @@ proto_library( "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", + "//envoy/extensions/compression/common/compressor/v3:pkg", "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/filter/udp/dns_filter/v3alpha:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", diff --git a/api/envoy/extensions/compression/common/compressor/v3/BUILD b/api/envoy/extensions/compression/common/compressor/v3/BUILD new file mode 100644 index 000000000000..ef3541ebcb1d --- /dev/null +++ b/api/envoy/extensions/compression/common/compressor/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/compression/common/compressor/v3/compressor.proto b/api/envoy/extensions/compression/common/compressor/v3/compressor.proto new file mode 100644 index 000000000000..f3259806b5f0 --- /dev/null +++ b/api/envoy/extensions/compression/common/compressor/v3/compressor.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package envoy.extensions.compression.common.compressor.v3; + +import "google/protobuf/any.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.compression.common.compressor.v3"; +option java_outer_classname = "CompressorProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: CompressorLibrary] + +message CompressorLibrary { + // Compressor library specific configuration. See the supported libraries for further + // documentation. + google.protobuf.Any typed_config = 1; +} diff --git a/api/envoy/extensions/filters/http/compressor/v3/BUILD b/api/envoy/extensions/filters/http/compressor/v3/BUILD index 8dc07faa8f9f..11ff14b0ad5f 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/BUILD +++ b/api/envoy/extensions/filters/http/compressor/v3/BUILD @@ -8,6 +8,7 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/compressor/v2:pkg", + "//envoy/extensions/compression/common/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 7faceb4849bf..3f07faec505f 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; +import "envoy/extensions/compression/common/compressor/v3/compressor.proto"; import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; @@ -54,11 +55,6 @@ message Compressor { // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. - CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; -} - -message CompressorLibrary { - // Compressor library specific configuration. See the supported libraries for further - // documentation. - google.protobuf.Any typed_config = 1; + compression.common.compressor.v3.CompressorLibrary compressor_library = 6 + [(validate.rules).message = {required: true}]; } diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 33fe07c815c9..7b736a4769df 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -44,6 +44,7 @@ proto_library( "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", + "//envoy/extensions/compression/common/compressor/v3:pkg", "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/filter/udp/dns_filter/v3alpha:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", diff --git a/docs/root/api-v3/config/compression/compression.rst b/docs/root/api-v3/config/compression/compression.rst index 80aa0ba927cc..ae7b178bfda0 100644 --- a/docs/root/api-v3/config/compression/compression.rst +++ b/docs/root/api-v3/config/compression/compression.rst @@ -5,4 +5,5 @@ Compression :glob: :maxdepth: 2 + ../../extensions/compression/common/*/v3/* ../../extensions/compression/gzip/*/v3/* diff --git a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD new file mode 100644 index 000000000000..ef3541ebcb1d --- /dev/null +++ b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto new file mode 100644 index 000000000000..f3259806b5f0 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package envoy.extensions.compression.common.compressor.v3; + +import "google/protobuf/any.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.compression.common.compressor.v3"; +option java_outer_classname = "CompressorProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: CompressorLibrary] + +message CompressorLibrary { + // Compressor library specific configuration. See the supported libraries for further + // documentation. + google.protobuf.Any typed_config = 1; +} diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD index 8dc07faa8f9f..11ff14b0ad5f 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD @@ -8,6 +8,7 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/compressor/v2:pkg", + "//envoy/extensions/compression/common/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 7faceb4849bf..3f07faec505f 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; +import "envoy/extensions/compression/common/compressor/v3/compressor.proto"; import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; @@ -54,11 +55,6 @@ message Compressor { // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. - CompressorLibrary compressor_library = 6 [(validate.rules).message = {required: true}]; -} - -message CompressorLibrary { - // Compressor library specific configuration. See the supported libraries for further - // documentation. - google.protobuf.Any typed_config = 1; + compression.common.compressor.v3.CompressorLibrary compressor_library = 6 + [(validate.rules).message = {required: true}]; } diff --git a/include/envoy/compression/compressor/BUILD b/include/envoy/compression/compressor/BUILD index 3e220abff580..443ab7ea3c77 100644 --- a/include/envoy/compression/compressor/BUILD +++ b/include/envoy/compression/compressor/BUILD @@ -8,11 +8,19 @@ load( envoy_package() +envoy_cc_library( + name = "compressor_interface", + hdrs = ["compressor.h"], + deps = [ + "//include/envoy/buffer:buffer_interface", + ], +) + envoy_cc_library( name = "compressor_factory_interface", hdrs = ["factory.h"], deps = [ - "//include/envoy/compressor:compressor_interface", + ":compressor_interface", ], ) diff --git a/include/envoy/compressor/compressor.h b/include/envoy/compression/compressor/compressor.h similarity index 93% rename from include/envoy/compressor/compressor.h rename to include/envoy/compression/compressor/compressor.h index 2ffe63efcad9..f236586d4ddf 100644 --- a/include/envoy/compressor/compressor.h +++ b/include/envoy/compression/compressor/compressor.h @@ -3,6 +3,7 @@ #include "envoy/buffer/buffer.h" namespace Envoy { +namespace Compression { namespace Compressor { /** @@ -29,4 +30,5 @@ class Compressor { using CompressorPtr = std::unique_ptr; } // namespace Compressor +} // namespace Compression } // namespace Envoy diff --git a/include/envoy/compression/compressor/factory.h b/include/envoy/compression/compressor/factory.h index a5f9996bc61f..4587e3a297b3 100644 --- a/include/envoy/compression/compressor/factory.h +++ b/include/envoy/compression/compressor/factory.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/compressor/compressor.h" +#include "envoy/compression/compressor/compressor.h" namespace Envoy { namespace Compression { @@ -10,7 +10,7 @@ class CompressorFactory { public: virtual ~CompressorFactory() = default; - virtual Envoy::Compressor::CompressorPtr createCompressor() PURE; + virtual CompressorPtr createCompressor() PURE; virtual const std::string& statsPrefix() const PURE; virtual const std::string& contentEncoding() const PURE; }; diff --git a/include/envoy/decompressor/BUILD b/include/envoy/compression/decompressor/BUILD similarity index 100% rename from include/envoy/decompressor/BUILD rename to include/envoy/compression/decompressor/BUILD diff --git a/include/envoy/decompressor/decompressor.h b/include/envoy/compression/decompressor/decompressor.h similarity index 92% rename from include/envoy/decompressor/decompressor.h rename to include/envoy/compression/decompressor/decompressor.h index d694aa50ca1c..154b0aaa742a 100644 --- a/include/envoy/decompressor/decompressor.h +++ b/include/envoy/compression/decompressor/decompressor.h @@ -3,6 +3,7 @@ #include "envoy/buffer/buffer.h" namespace Envoy { +namespace Compression { namespace Decompressor { /** @@ -22,4 +23,5 @@ class Decompressor { }; } // namespace Decompressor +} // namespace Compression } // namespace Envoy diff --git a/source/common/common/BUILD b/source/common/common/BUILD index af25042d18c5..ccc60a82c2fc 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -420,16 +420,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "zlib_base_lib", - srcs = ["zlib/base.cc"], - hdrs = ["zlib/base.h"], - external_deps = ["zlib"], - deps = [ - "//source/common/buffer:buffer_lib", - ], -) - envoy_cc_library( name = "statusor_lib", hdrs = ["statusor.h"], diff --git a/source/common/compressor/BUILD b/source/common/compressor/BUILD deleted file mode 100644 index d452e1c968f7..000000000000 --- a/source/common/compressor/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load( - "//bazel:envoy_build_system.bzl", - "envoy_cc_library", - "envoy_package", -) - -envoy_package() - -envoy_cc_library( - name = "compressor_lib", - srcs = ["zlib_compressor_impl.cc"], - hdrs = ["zlib_compressor_impl.h"], - external_deps = ["zlib"], - deps = [ - "//include/envoy/compressor:compressor_interface", - "//source/common/buffer:buffer_lib", - "//source/common/common:assert_lib", - "//source/common/common:zlib_base_lib", - ], -) diff --git a/source/extensions/compression/common/BUILD b/source/extensions/compression/common/compressor/BUILD similarity index 91% rename from source/extensions/compression/common/BUILD rename to source/extensions/compression/common/compressor/BUILD index ad0fe31a359f..cd4d685b48aa 100644 --- a/source/extensions/compression/common/BUILD +++ b/source/extensions/compression/common/compressor/BUILD @@ -10,7 +10,7 @@ envoy_package() envoy_cc_library( name = "compressor_factory_base_lib", - hdrs = ["compressor_factory_base.h"], + hdrs = ["factory_base.h"], deps = [ "//include/envoy/compression/compressor:compressor_factory_interface", "//include/envoy/compression/compressor:compressor_library_config_interface", diff --git a/source/extensions/compression/common/compressor_factory_base.h b/source/extensions/compression/common/compressor/factory_base.h similarity index 96% rename from source/extensions/compression/common/compressor_factory_base.h rename to source/extensions/compression/common/compressor/factory_base.h index 0f6095f5e151..fe2ddefb9c32 100644 --- a/source/extensions/compression/common/compressor_factory_base.h +++ b/source/extensions/compression/common/compressor/factory_base.h @@ -8,6 +8,7 @@ namespace Envoy { namespace Extensions { namespace Compression { namespace Common { +namespace Compressor { template class CompressorLibraryFactoryBase @@ -38,6 +39,7 @@ class CompressorLibraryFactoryBase const std::string name_; }; +} // namespace Compressor } // namespace Common } // namespace Compression } // namespace Extensions diff --git a/include/envoy/compressor/BUILD b/source/extensions/compression/gzip/common/BUILD similarity index 54% rename from include/envoy/compressor/BUILD rename to source/extensions/compression/gzip/common/BUILD index 9b3b8f43e47d..7c9583e7985f 100644 --- a/include/envoy/compressor/BUILD +++ b/source/extensions/compression/gzip/common/BUILD @@ -9,9 +9,11 @@ load( envoy_package() envoy_cc_library( - name = "compressor_interface", - hdrs = ["compressor.h"], + name = "gzip_base_lib", + srcs = ["base.cc"], + hdrs = ["base.h"], + external_deps = ["zlib"], deps = [ - "//include/envoy/buffer:buffer_interface", + "//source/common/buffer:buffer_lib", ], ) diff --git a/source/common/common/zlib/base.cc b/source/extensions/compression/gzip/common/base.cc similarity index 73% rename from source/common/common/zlib/base.cc rename to source/extensions/compression/gzip/common/base.cc index 5336f35f8735..1bd594932024 100644 --- a/source/common/common/zlib/base.cc +++ b/source/extensions/compression/gzip/common/base.cc @@ -1,7 +1,10 @@ -#include "common/common/zlib/base.h" +#include "extensions/compression/gzip/common/base.h" namespace Envoy { -namespace Zlib { +namespace Extensions { +namespace Compression { +namespace Gzip { +namespace Common { Base::Base(uint64_t chunk_size, std::function zstream_deleter) : chunk_size_{chunk_size}, chunk_char_ptr_(new unsigned char[chunk_size]), @@ -20,5 +23,8 @@ void Base::updateOutput(Buffer::Instance& output_buffer) { zstream_ptr_->next_out = chunk_char_ptr_.get(); } -} // namespace Zlib +} // namespace Common +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/source/common/common/zlib/base.h b/source/extensions/compression/gzip/common/base.h similarity index 84% rename from source/common/common/zlib/base.h rename to source/extensions/compression/gzip/common/base.h index 4f427fb90985..720ebc68f118 100644 --- a/source/common/common/zlib/base.h +++ b/source/extensions/compression/gzip/common/base.h @@ -7,7 +7,10 @@ #include "zlib.h" namespace Envoy { -namespace Zlib { +namespace Extensions { +namespace Compression { +namespace Gzip { +namespace Common { /** * Shared code between the compressor and the decompressor. @@ -36,5 +39,8 @@ class Base { const std::unique_ptr> zstream_ptr_; }; -} // namespace Zlib +} // namespace Common +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/source/extensions/compression/gzip/compressor/BUILD b/source/extensions/compression/gzip/compressor/BUILD index 5c8a3276d5d4..54f476883068 100644 --- a/source/extensions/compression/gzip/compressor/BUILD +++ b/source/extensions/compression/gzip/compressor/BUILD @@ -3,20 +3,34 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", "envoy_cc_extension", + "envoy_cc_library", "envoy_package", ) envoy_package() +envoy_cc_library( + name = "compressor_lib", + srcs = ["zlib_compressor_impl.cc"], + hdrs = ["zlib_compressor_impl.h"], + external_deps = ["zlib"], + deps = [ + "//include/envoy/compression/compressor:compressor_interface", + "//source/common/buffer:buffer_lib", + "//source/common/common:assert_lib", + "//source/extensions/compression/gzip/common:gzip_base_lib", + ], +) + envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], security_posture = "robust_to_untrusted_downstream", deps = [ - "//source/common/compressor:compressor_lib", + ":compressor_lib", "//source/common/http:headers_lib", - "//source/extensions/compression/common:compressor_factory_base_lib", + "//source/extensions/compression/common/compressor:compressor_factory_base_lib", "//source/extensions/filters/http:well_known_names", "@envoy_api//envoy/extensions/compression/gzip/compressor/v3:pkg_cc_proto", ], diff --git a/source/extensions/compression/gzip/compressor/config.cc b/source/extensions/compression/gzip/compressor/config.cc index fd529b3b5057..9d37441547f4 100644 --- a/source/extensions/compression/gzip/compressor/config.cc +++ b/source/extensions/compression/gzip/compressor/config.cc @@ -29,53 +29,52 @@ GzipCompressorFactory::GzipCompressorFactory( GzipHeaderValue), chunk_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(gzip, chunk_size, DefaultChunkSize)) {} -Envoy::Compressor::ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::compressionLevelEnum( +ZlibCompressorImpl::CompressionLevel GzipCompressorFactory::compressionLevelEnum( envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionLevel compression_level) { switch (compression_level) { case envoy::extensions::compression::gzip::compressor::v3::Gzip::BEST_SPEED: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Speed; + return ZlibCompressorImpl::CompressionLevel::Speed; case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_2: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level2; + return ZlibCompressorImpl::CompressionLevel::Level2; case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_3: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level3; + return ZlibCompressorImpl::CompressionLevel::Level3; case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_4: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level4; + return ZlibCompressorImpl::CompressionLevel::Level4; case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_5: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level5; + return ZlibCompressorImpl::CompressionLevel::Level5; case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_6: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level6; + return ZlibCompressorImpl::CompressionLevel::Level6; case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_7: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level7; + return ZlibCompressorImpl::CompressionLevel::Level7; case envoy::extensions::compression::gzip::compressor::v3::Gzip::COMPRESSION_LEVEL_8: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level8; + return ZlibCompressorImpl::CompressionLevel::Level8; case envoy::extensions::compression::gzip::compressor::v3::Gzip::BEST_COMPRESSION: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best; + return ZlibCompressorImpl::CompressionLevel::Best; default: - return Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard; + return ZlibCompressorImpl::CompressionLevel::Standard; } } -Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy -GzipCompressorFactory::compressionStrategyEnum( +ZlibCompressorImpl::CompressionStrategy GzipCompressorFactory::compressionStrategyEnum( envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionStrategy compression_strategy) { switch (compression_strategy) { case envoy::extensions::compression::gzip::compressor::v3::Gzip::FILTERED: - return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered; + return ZlibCompressorImpl::CompressionStrategy::Filtered; case envoy::extensions::compression::gzip::compressor::v3::Gzip::FIXED: - return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed; + return ZlibCompressorImpl::CompressionStrategy::Fixed; case envoy::extensions::compression::gzip::compressor::v3::Gzip::HUFFMAN_ONLY: - return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman; + return ZlibCompressorImpl::CompressionStrategy::Huffman; case envoy::extensions::compression::gzip::compressor::v3::Gzip::RLE: - return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle; + return ZlibCompressorImpl::CompressionStrategy::Rle; default: - return Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard; + return ZlibCompressorImpl::CompressionStrategy::Standard; } } -Envoy::Compressor::CompressorPtr GzipCompressorFactory::createCompressor() { - auto compressor = std::make_unique(chunk_size_); +Envoy::Compression::Compressor::CompressorPtr GzipCompressorFactory::createCompressor() { + auto compressor = std::make_unique(chunk_size_); compressor->init(compression_level_, compression_strategy_, window_bits_, memory_level_); return compressor; } diff --git a/source/extensions/compression/gzip/compressor/config.h b/source/extensions/compression/gzip/compressor/config.h index 024eb9a6c93c..e5d8325a3b8f 100644 --- a/source/extensions/compression/gzip/compressor/config.h +++ b/source/extensions/compression/gzip/compressor/config.h @@ -4,10 +4,10 @@ #include "envoy/extensions/compression/gzip/compressor/v3/gzip.pb.h" #include "envoy/extensions/compression/gzip/compressor/v3/gzip.pb.validate.h" -#include "common/compressor/zlib_compressor_impl.h" #include "common/http/headers.h" -#include "extensions/compression/common/compressor_factory_base.h" +#include "extensions/compression/common/compressor/factory_base.h" +#include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" #include "extensions/filters/http/well_known_names.h" namespace Envoy { @@ -29,7 +29,7 @@ class GzipCompressorFactory : public Envoy::Compression::Compressor::CompressorF public: GzipCompressorFactory(const envoy::extensions::compression::gzip::compressor::v3::Gzip& gzip); - Envoy::Compressor::CompressorPtr createCompressor() override; + Envoy::Compression::Compressor::CompressorPtr createCompressor() override; const std::string& statsPrefix() const override { return gzipStatsPrefix(); } const std::string& contentEncoding() const override { return Http::Headers::get().ContentEncodingValues.Gzip; @@ -38,15 +38,15 @@ class GzipCompressorFactory : public Envoy::Compression::Compressor::CompressorF private: friend class GzipTest; - static Envoy::Compressor::ZlibCompressorImpl::CompressionLevel + static ZlibCompressorImpl::CompressionLevel compressionLevelEnum(envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionLevel compression_level); - static Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategyEnum( + static ZlibCompressorImpl::CompressionStrategy compressionStrategyEnum( envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionStrategy compression_strategy); - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compression_level_; - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compression_strategy_; + ZlibCompressorImpl::CompressionLevel compression_level_; + ZlibCompressorImpl::CompressionStrategy compression_strategy_; const int32_t memory_level_; const int32_t window_bits_; @@ -54,7 +54,7 @@ class GzipCompressorFactory : public Envoy::Compression::Compressor::CompressorF }; class GzipCompressorLibraryFactory - : public Common::CompressorLibraryFactoryBase< + : public Compression::Common::Compressor::CompressorLibraryFactoryBase< envoy::extensions::compression::gzip::compressor::v3::Gzip> { public: GzipCompressorLibraryFactory() : CompressorLibraryFactoryBase(gzipExtensionName()) {} diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.cc similarity index 83% rename from source/common/compressor/zlib_compressor_impl.cc rename to source/extensions/compression/gzip/compressor/zlib_compressor_impl.cc index 2f44a5da1a90..39f5b3e5d729 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.cc @@ -1,4 +1,4 @@ -#include "common/compressor/zlib_compressor_impl.h" +#include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" #include @@ -9,12 +9,15 @@ #include "absl/container/fixed_array.h" namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { namespace Compressor { ZlibCompressorImpl::ZlibCompressorImpl() : ZlibCompressorImpl(4096) {} ZlibCompressorImpl::ZlibCompressorImpl(uint64_t chunk_size) - : Zlib::Base(chunk_size, [](z_stream* z) { + : Common::Base(chunk_size, [](z_stream* z) { deflateEnd(z); delete z; }) { @@ -34,7 +37,8 @@ void ZlibCompressorImpl::init(CompressionLevel comp_level, CompressionStrategy c initialized_ = true; } -void ZlibCompressorImpl::compress(Buffer::Instance& buffer, State state) { +void ZlibCompressorImpl::compress(Buffer::Instance& buffer, + Envoy::Compression::Compressor::State state) { for (const Buffer::RawSlice& input_slice : buffer.getRawSlices()) { zstream_ptr_->avail_in = input_slice.len_; zstream_ptr_->next_in = static_cast(input_slice.mem_); @@ -46,7 +50,7 @@ void ZlibCompressorImpl::compress(Buffer::Instance& buffer, State state) { buffer.drain(input_slice.len_); } - process(buffer, state == State::Finish ? Z_FINISH : Z_SYNC_FLUSH); + process(buffer, state == Envoy::Compression::Compressor::State::Finish ? Z_FINISH : Z_SYNC_FLUSH); } bool ZlibCompressorImpl::deflateNext(int64_t flush_state) { @@ -81,4 +85,7 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ } } // namespace Compressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h similarity index 87% rename from source/common/compressor/zlib_compressor_impl.h rename to source/extensions/compression/gzip/compressor/zlib_compressor_impl.h index 2610ea0af775..f182a6ff9d40 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h @@ -1,18 +1,21 @@ #pragma once -#include "envoy/compressor/compressor.h" +#include "envoy/compression/compressor/compressor.h" -#include "common/common/zlib/base.h" +#include "extensions/compression/gzip/common/base.h" #include "zlib.h" namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { namespace Compressor { /** * Implementation of compressor's interface. */ -class ZlibCompressorImpl : public Zlib::Base, public Compressor { +class ZlibCompressorImpl : public Common::Base, public Envoy::Compression::Compressor::Compressor { public: ZlibCompressorImpl(); @@ -80,7 +83,7 @@ class ZlibCompressorImpl : public Zlib::Base, public Compressor { uint64_t memory_level); // Compressor - void compress(Buffer::Instance& buffer, State state) override; + void compress(Buffer::Instance& buffer, Envoy::Compression::Compressor::State state) override; private: bool deflateNext(int64_t flush_state); @@ -88,4 +91,7 @@ class ZlibCompressorImpl : public Zlib::Base, public Compressor { }; } // namespace Compressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/source/common/decompressor/BUILD b/source/extensions/compression/gzip/decompressor/BUILD similarity index 76% rename from source/common/decompressor/BUILD rename to source/extensions/compression/gzip/decompressor/BUILD index dfdf8f9b90ed..323930ba9a7a 100644 --- a/source/common/decompressor/BUILD +++ b/source/extensions/compression/gzip/decompressor/BUILD @@ -14,10 +14,10 @@ envoy_cc_library( hdrs = ["zlib_decompressor_impl.h"], external_deps = ["zlib"], deps = [ - "//include/envoy/decompressor:decompressor_interface", + "//include/envoy/compression/decompressor:decompressor_interface", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", - "//source/common/common:zlib_base_lib", + "//source/extensions/compression/gzip/common:gzip_base_lib", ], ) diff --git a/source/common/decompressor/zlib_decompressor_impl.cc b/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc similarity index 89% rename from source/common/decompressor/zlib_decompressor_impl.cc rename to source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc index 55dffc6d3609..42e7d12e1137 100644 --- a/source/common/decompressor/zlib_decompressor_impl.cc +++ b/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc @@ -1,4 +1,4 @@ -#include "common/decompressor/zlib_decompressor_impl.h" +#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include @@ -9,12 +9,15 @@ #include "absl/container/fixed_array.h" namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { namespace Decompressor { ZlibDecompressorImpl::ZlibDecompressorImpl() : ZlibDecompressorImpl(4096) {} ZlibDecompressorImpl::ZlibDecompressorImpl(uint64_t chunk_size) - : Zlib::Base(chunk_size, [](z_stream* z) { + : Common::Base(chunk_size, [](z_stream* z) { inflateEnd(z); delete z; }) { @@ -75,4 +78,7 @@ bool ZlibDecompressorImpl::inflateNext() { } } // namespace Decompressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/source/common/decompressor/zlib_decompressor_impl.h b/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.h similarity index 81% rename from source/common/decompressor/zlib_decompressor_impl.h rename to source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.h index 8d5627fc6c31..b290fb851915 100644 --- a/source/common/decompressor/zlib_decompressor_impl.h +++ b/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.h @@ -1,20 +1,24 @@ #pragma once -#include "envoy/decompressor/decompressor.h" +#include "envoy/compression/decompressor/decompressor.h" #include "common/common/logger.h" -#include "common/common/zlib/base.h" + +#include "extensions/compression/gzip/common/base.h" #include "zlib.h" namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { namespace Decompressor { /** * Implementation of decompressor's interface. */ -class ZlibDecompressorImpl : public Zlib::Base, - public Decompressor, +class ZlibDecompressorImpl : public Common::Base, + public Envoy::Compression::Decompressor::Decompressor, public Logger::Loggable { public: ZlibDecompressorImpl(); @@ -49,4 +53,7 @@ class ZlibDecompressorImpl : public Zlib::Base, }; } // namespace Decompressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/source/extensions/filters/http/common/compressor/BUILD b/source/extensions/filters/http/common/compressor/BUILD index 60995f895b0f..7058ab2fc58b 100644 --- a/source/extensions/filters/http/common/compressor/BUILD +++ b/source/extensions/filters/http/common/compressor/BUILD @@ -14,7 +14,7 @@ envoy_cc_library( srcs = ["compressor.cc"], hdrs = ["compressor.h"], deps = [ - "//include/envoy/compressor:compressor_interface", + "//include/envoy/compression/compressor:compressor_interface", "//include/envoy/stats:stats_macros", "//include/envoy/stream_info:filter_state_interface", "//source/common/buffer:buffer_lib", diff --git a/source/extensions/filters/http/common/compressor/compressor.cc b/source/extensions/filters/http/common/compressor/compressor.cc index 7c6ac05893b3..ac2fc769ec0f 100644 --- a/source/extensions/filters/http/common/compressor/compressor.cc +++ b/source/extensions/filters/http/common/compressor/compressor.cc @@ -121,7 +121,8 @@ Http::FilterHeadersStatus CompressorFilter::encodeHeaders(Http::ResponseHeaderMa Http::FilterDataStatus CompressorFilter::encodeData(Buffer::Instance& data, bool end_stream) { if (!skip_compression_) { config_->stats().total_uncompressed_bytes_.add(data.length()); - compressor_->compress(data, end_stream ? Compressor::State::Finish : Compressor::State::Flush); + compressor_->compress(data, end_stream ? Envoy::Compression::Compressor::State::Finish + : Envoy::Compression::Compressor::State::Flush); config_->stats().total_compressed_bytes_.add(data.length()); } return Http::FilterDataStatus::Continue; @@ -130,7 +131,7 @@ Http::FilterDataStatus CompressorFilter::encodeData(Buffer::Instance& data, bool Http::FilterTrailersStatus CompressorFilter::encodeTrailers(Http::ResponseTrailerMap&) { if (!skip_compression_) { Buffer::OwnedImpl empty_buffer; - compressor_->compress(empty_buffer, Compressor::State::Finish); + compressor_->compress(empty_buffer, Envoy::Compression::Compressor::State::Finish); config_->stats().total_compressed_bytes_.add(empty_buffer.length()); encoder_callbacks_->addEncodedData(empty_buffer, true); } diff --git a/source/extensions/filters/http/common/compressor/compressor.h b/source/extensions/filters/http/common/compressor/compressor.h index 4a15782b8312..844719a33466 100644 --- a/source/extensions/filters/http/common/compressor/compressor.h +++ b/source/extensions/filters/http/common/compressor/compressor.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/compressor/compressor.h" +#include "envoy/compression/compressor/compressor.h" #include "envoy/extensions/filters/http/compressor/v3/compressor.pb.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" @@ -62,7 +62,7 @@ class CompressorFilterConfig { CompressorFilterConfig() = delete; virtual ~CompressorFilterConfig() = default; - virtual Envoy::Compressor::CompressorPtr makeCompressor() PURE; + virtual Envoy::Compression::Compressor::CompressorPtr makeCompressor() PURE; bool enabled() const { return enabled_.enabled(); } const CompressorStats& stats() { return stats_; } @@ -149,7 +149,7 @@ class CompressorFilter : public Http::PassThroughFilter { bool shouldCompress(const EncodingDecision& decision) const; bool skip_compression_; - Envoy::Compressor::CompressorPtr compressor_; + Envoy::Compression::Compressor::CompressorPtr compressor_; const CompressorFilterConfigSharedPtr config_; std::unique_ptr accept_encoding_; }; diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index 057f8af67e45..bb37336fbbd0 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -14,7 +14,7 @@ CompressorFilterConfig::CompressorFilterConfig( compressor_factory->contentEncoding()), compressor_factory_(std::move(compressor_factory)) {} -Envoy::Compressor::CompressorPtr CompressorFilterConfig::makeCompressor() { +Envoy::Compression::Compressor::CompressorPtr CompressorFilterConfig::makeCompressor() { return compressor_factory_->createCompressor(); } diff --git a/source/extensions/filters/http/compressor/compressor_filter.h b/source/extensions/filters/http/compressor/compressor_filter.h index 9e648fece872..8d7347847f79 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.h +++ b/source/extensions/filters/http/compressor/compressor_filter.h @@ -23,7 +23,7 @@ class CompressorFilterConfig : public Common::Compressors::CompressorFilterConfi const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, Envoy::Compression::Compressor::CompressorFactoryPtr compressor_factory); - Envoy::Compressor::CompressorPtr makeCompressor() override; + Envoy::Compression::Compressor::CompressorPtr makeCompressor() override; private: const Envoy::Compression::Compressor::CompressorFactoryPtr compressor_factory_; diff --git a/source/extensions/filters/http/gzip/BUILD b/source/extensions/filters/http/gzip/BUILD index 168783406be1..b27fe1fef620 100644 --- a/source/extensions/filters/http/gzip/BUILD +++ b/source/extensions/filters/http/gzip/BUILD @@ -17,9 +17,9 @@ envoy_cc_library( srcs = ["gzip_filter.cc"], hdrs = ["gzip_filter.h"], deps = [ - "//source/common/compressor:compressor_lib", "//source/common/http:headers_lib", "//source/common/protobuf", + "//source/extensions/compression/gzip/compressor:compressor_lib", "//source/extensions/filters/http/common/compressor:compressor_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/gzip/v3:pkg_cc_proto", diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index 8daaf6fa237a..a7d9cb0bc14b 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -32,35 +32,37 @@ GzipFilterConfig::GzipFilterConfig(const envoy::extensions::filters::http::gzip: memory_level_(memoryLevelUint(gzip.memory_level().value())), window_bits_(windowBitsUint(gzip.window_bits().value())) {} -Compressor::CompressorPtr GzipFilterConfig::makeCompressor() { - auto compressor = std::make_unique(); +Envoy::Compression::Compressor::CompressorPtr GzipFilterConfig::makeCompressor() { + auto compressor = std::make_unique(); compressor->init(compressionLevel(), compressionStrategy(), windowBits(), memoryLevel()); return compressor; } -Compressor::ZlibCompressorImpl::CompressionLevel GzipFilterConfig::compressionLevelEnum( +Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel +GzipFilterConfig::compressionLevelEnum( envoy::extensions::filters::http::gzip::v3::Gzip::CompressionLevel::Enum compression_level) { switch (compression_level) { case envoy::extensions::filters::http::gzip::v3::Gzip::CompressionLevel::BEST: - return Compressor::ZlibCompressorImpl::CompressionLevel::Best; + return Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Best; case envoy::extensions::filters::http::gzip::v3::Gzip::CompressionLevel::SPEED: - return Compressor::ZlibCompressorImpl::CompressionLevel::Speed; + return Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Speed; default: - return Compressor::ZlibCompressorImpl::CompressionLevel::Standard; + return Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard; } } -Compressor::ZlibCompressorImpl::CompressionStrategy GzipFilterConfig::compressionStrategyEnum( +Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy +GzipFilterConfig::compressionStrategyEnum( envoy::extensions::filters::http::gzip::v3::Gzip::CompressionStrategy compression_strategy) { switch (compression_strategy) { case envoy::extensions::filters::http::gzip::v3::Gzip::RLE: - return Compressor::ZlibCompressorImpl::CompressionStrategy::Rle; + return Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle; case envoy::extensions::filters::http::gzip::v3::Gzip::FILTERED: - return Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered; + return Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered; case envoy::extensions::filters::http::gzip::v3::Gzip::HUFFMAN: - return Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman; + return Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman; default: - return Compressor::ZlibCompressorImpl::CompressionStrategy::Standard; + return Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard; } } diff --git a/source/extensions/filters/http/gzip/gzip_filter.h b/source/extensions/filters/http/gzip/gzip_filter.h index a996dd4f7937..57dfea9b0d84 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.h +++ b/source/extensions/filters/http/gzip/gzip_filter.h @@ -2,8 +2,7 @@ #include "envoy/extensions/filters/http/gzip/v3/gzip.pb.h" -#include "common/compressor/zlib_compressor_impl.h" - +#include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" #include "extensions/filters/http/common/compressor/compressor.h" namespace Envoy { @@ -20,12 +19,13 @@ class GzipFilterConfig : public Common::Compressors::CompressorFilterConfig { GzipFilterConfig(const envoy::extensions::filters::http::gzip::v3::Gzip& gzip, const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime); - Compressor::CompressorPtr makeCompressor() override; + Envoy::Compression::Compressor::CompressorPtr makeCompressor() override; - Compressor::ZlibCompressorImpl::CompressionLevel compressionLevel() const { + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel compressionLevel() const { return compression_level_; } - Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategy() const { + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy + compressionStrategy() const { return compression_strategy_; } @@ -33,9 +33,10 @@ class GzipFilterConfig : public Common::Compressors::CompressorFilterConfig { uint64_t windowBits() const { return window_bits_; } private: - static Compressor::ZlibCompressorImpl::CompressionLevel compressionLevelEnum( + static Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel compressionLevelEnum( envoy::extensions::filters::http::gzip::v3::Gzip::CompressionLevel::Enum compression_level); - static Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategyEnum( + static Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy + compressionStrategyEnum( envoy::extensions::filters::http::gzip::v3::Gzip::CompressionStrategy compression_strategy); static uint64_t memoryLevelUint(Protobuf::uint32 level); @@ -45,8 +46,8 @@ class GzipFilterConfig : public Common::Compressors::CompressorFilterConfig { static const envoy::extensions::filters::http::compressor::v3::Compressor compressorConfig(const envoy::extensions::filters::http::gzip::v3::Gzip& gzip); - Compressor::ZlibCompressorImpl::CompressionLevel compression_level_; - Compressor::ZlibCompressorImpl::CompressionStrategy compression_strategy_; + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel compression_level_; + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy compression_strategy_; int32_t memory_level_; int32_t window_bits_; diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index ad8e7885cd1c..f7e99eaf9ffe 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -157,8 +157,8 @@ envoy_cc_fuzz_test( ":stat_test_utility_lib", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", - "//source/common/decompressor:decompressor_lib", "//source/common/stats:symbol_table_lib", + "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//test/fuzz:utility_lib", ], ) diff --git a/test/extensions/compression/gzip/BUILD b/test/extensions/compression/gzip/BUILD deleted file mode 100644 index c36c79504e58..000000000000 --- a/test/extensions/compression/gzip/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load( - "//bazel:envoy_build_system.bzl", - "envoy_package", -) -load( - "//test/extensions:extensions_build_system.bzl", - "envoy_extension_cc_test", -) - -envoy_package() - -envoy_extension_cc_test( - name = "gzip_compressor_library_test", - srcs = ["gzip_compressor_library_test.cc"], - extension_name = "envoy.compression.gzip.compressor", - deps = [ - "//source/extensions/compression/gzip/compressor:config", - "//test/test_common:utility_lib", - ], -) - -envoy_extension_cc_test( - name = "gzip_compressor_integration_test", - srcs = ["gzip_filter_test.cc"], - extension_name = "envoy.compression.gzip.compressor", - deps = [ - "//source/common/decompressor:decompressor_lib", - "//source/extensions/compression/gzip/compressor:config", - "//source/extensions/filters/http/compressor:compressor_filter_lib", - "//test/mocks/http:http_mocks", - ], -) diff --git a/test/common/compressor/BUILD b/test/extensions/compression/gzip/compressor/BUILD similarity index 51% rename from test/common/compressor/BUILD rename to test/extensions/compression/gzip/compressor/BUILD index e58b835ab69a..874f4f869dcd 100644 --- a/test/common/compressor/BUILD +++ b/test/extensions/compression/gzip/compressor/BUILD @@ -6,6 +6,10 @@ load( "envoy_cc_test", "envoy_package", ) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) envoy_package() @@ -16,8 +20,8 @@ envoy_cc_fuzz_test( deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", - "//source/common/compressor:compressor_lib", - "//source/common/decompressor:decompressor_lib", + "//source/extensions/compression/gzip/compressor:compressor_lib", + "//source/extensions/compression/gzip/decompressor:decompressor_lib", ], ) @@ -27,7 +31,17 @@ envoy_cc_test( deps = [ "//source/common/common:assert_lib", "//source/common/common:hex_lib", - "//source/common/compressor:compressor_lib", + "//source/extensions/compression/gzip/compressor:compressor_lib", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "gzip_compressor_library_test", + srcs = ["gzip_compressor_library_test.cc"], + extension_name = "envoy.compression.gzip.compressor", + deps = [ + "//source/extensions/compression/gzip/compressor:config", "//test/test_common:utility_lib", ], ) diff --git a/test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 b/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 similarity index 100% rename from test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 rename to test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 diff --git a/test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 b/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 similarity index 100% rename from test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 rename to test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 diff --git a/test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 b/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 similarity index 100% rename from test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 rename to test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 diff --git a/test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 b/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 similarity index 100% rename from test/common/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 rename to test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 diff --git a/test/common/compressor/compressor_corpus/empty b/test/extensions/compression/gzip/compressor/compressor_corpus/empty similarity index 100% rename from test/common/compressor/compressor_corpus/empty rename to test/extensions/compression/gzip/compressor/compressor_corpus/empty diff --git a/test/common/compressor/compressor_corpus/noise b/test/extensions/compression/gzip/compressor/compressor_corpus/noise similarity index 100% rename from test/common/compressor/compressor_corpus/noise rename to test/extensions/compression/gzip/compressor/compressor_corpus/noise diff --git a/test/common/compressor/compressor_corpus/simple b/test/extensions/compression/gzip/compressor/compressor_corpus/simple similarity index 100% rename from test/common/compressor/compressor_corpus/simple rename to test/extensions/compression/gzip/compressor/compressor_corpus/simple diff --git a/test/common/compressor/compressor_fuzz_test.cc b/test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc similarity index 86% rename from test/common/compressor/compressor_fuzz_test.cc rename to test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc index 1c28ac5bcc5c..da76007c8989 100644 --- a/test/common/compressor/compressor_fuzz_test.cc +++ b/test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc @@ -1,11 +1,15 @@ #include "common/buffer/buffer_impl.h" #include "common/common/assert.h" -#include "common/compressor/zlib_compressor_impl.h" -#include "common/decompressor/zlib_decompressor_impl.h" + +#include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" +#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "test/fuzz/fuzz_runner.h" namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { namespace Compressor { namespace Fuzz { @@ -61,7 +65,8 @@ DEFINE_FUZZER(const uint8_t* buf, size_t len) { full_input.add(next_data); Buffer::OwnedImpl buffer{next_data.data(), next_data.size()}; provider_empty = provider.remaining_bytes() == 0; - compressor.compress(buffer, provider_empty ? State::Finish : State::Flush); + compressor.compress(buffer, provider_empty ? Envoy::Compression::Compressor::State::Finish + : Envoy::Compression::Compressor::State::Flush); decompressor.decompress(buffer, full_output); } RELEASE_ASSERT(full_input.toString() == full_output.toString(), ""); @@ -70,4 +75,7 @@ DEFINE_FUZZER(const uint8_t* buf, size_t len) { } // namespace Fuzz } // namespace Compressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc b/test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc new file mode 100644 index 000000000000..78395614a218 --- /dev/null +++ b/test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc @@ -0,0 +1,107 @@ +#include "extensions/compression/gzip/compressor/config.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { +namespace Compressor { + +class GzipTest : public testing::Test { +protected: + void SetUp() override { setUpGzip("{}"); } + + // GzipTest Helpers + void setUpGzip(std::string&& json) { + envoy::extensions::compression::gzip::compressor::v3::Gzip gzip; + TestUtility::loadFromJson(json, gzip); + factory_ = std::make_unique(gzip); + } + + ZlibCompressorImpl::CompressionLevel compressionLevel() const { + return factory_->compression_level_; + } + + ZlibCompressorImpl::CompressionStrategy compressionStrategy() const { + return factory_->compression_strategy_; + } + + int32_t memoryLevel() const { return factory_->memory_level_; } + int32_t windowBits() const { return factory_->window_bits_; } + int32_t chunkSize() const { return factory_->chunk_size_; } + + void expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy strategy, + absl::string_view strategy_name, + ZlibCompressorImpl::CompressionLevel level, + absl::string_view level_name) { + setUpGzip(fmt::format( + R"EOF({{"compression_strategy": "{}", "compression_level": "{}", "memory_level": 6, "window_bits": 27, "chunk_size": 10000}})EOF", + strategy_name, level_name)); + EXPECT_EQ(strategy, compressionStrategy()); + EXPECT_EQ(level, compressionLevel()); + EXPECT_EQ(6, memoryLevel()); + EXPECT_EQ(27, windowBits()); + EXPECT_EQ(10000, chunkSize()); + } + + std::unique_ptr factory_; +}; + +// Default config values. +TEST_F(GzipTest, DefaultConfigValues) { + EXPECT_EQ(5, memoryLevel()); + EXPECT_EQ(28, windowBits()); + EXPECT_EQ(4096, chunkSize()); + EXPECT_EQ(ZlibCompressorImpl::CompressionStrategy::Standard, compressionStrategy()); + EXPECT_EQ(ZlibCompressorImpl::CompressionLevel::Standard, compressionLevel()); +} + +TEST_F(GzipTest, AvailableCombinationCompressionStrategyAndLevelConfig) { + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Filtered, + "FILTERED", ZlibCompressorImpl::CompressionLevel::Best, + "BEST_COMPRESSION"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Huffman, + "HUFFMAN_ONLY", ZlibCompressorImpl::CompressionLevel::Best, + "BEST_COMPRESSION"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Rle, "RLE", + ZlibCompressorImpl::CompressionLevel::Speed, "BEST_SPEED"); + expectValidCompressionStrategyAndLevel( + ZlibCompressorImpl::CompressionStrategy::Standard, "DEFAULT_STRATEGY", + ZlibCompressorImpl::CompressionLevel::Standard, "DEFAULT_COMPRESSION"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level1, + "COMPRESSION_LEVEL_1"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level2, + "COMPRESSION_LEVEL_2"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level3, + "COMPRESSION_LEVEL_3"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level4, + "COMPRESSION_LEVEL_4"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level5, + "COMPRESSION_LEVEL_5"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level6, + "COMPRESSION_LEVEL_6"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level7, + "COMPRESSION_LEVEL_7"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level8, + "COMPRESSION_LEVEL_8"); + expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", + ZlibCompressorImpl::CompressionLevel::Level9, + "COMPRESSION_LEVEL_9"); +} + +} // namespace Compressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/extensions/compression/gzip/compressor/zlib_compressor_impl_test.cc similarity index 89% rename from test/common/compressor/zlib_compressor_impl_test.cc rename to test/extensions/compression/gzip/compressor/zlib_compressor_impl_test.cc index 3e2db26f4d43..9d3c0fb80644 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/extensions/compression/gzip/compressor/zlib_compressor_impl_test.cc @@ -1,6 +1,7 @@ #include "common/buffer/buffer_impl.h" #include "common/common/hex.h" -#include "common/compressor/zlib_compressor_impl.h" + +#include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" #include "test/test_common/utility.h" @@ -8,6 +9,9 @@ #include "gtest/gtest.h" namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { namespace Compressor { namespace { @@ -70,8 +74,12 @@ class ZlibCompressorImplTester : public ZlibCompressorImpl { public: ZlibCompressorImplTester() = default; ZlibCompressorImplTester(uint64_t chunk_size) : ZlibCompressorImpl(chunk_size) {} - void compressThenFlush(Buffer::OwnedImpl& buffer) { compress(buffer, State::Flush); } - void finish(Buffer::OwnedImpl& buffer) { compress(buffer, State::Finish); } + void compressThenFlush(Buffer::OwnedImpl& buffer) { + compress(buffer, Envoy::Compression::Compressor::State::Flush); + } + void finish(Buffer::OwnedImpl& buffer) { + compress(buffer, Envoy::Compression::Compressor::State::Finish); + } }; class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { @@ -121,9 +129,9 @@ TEST_F(ZlibCompressorImplTest, CallingChecksum) { ZlibCompressorImplTester compressor; EXPECT_EQ(0, compressor.checksum()); - compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, - gzip_window_bits, memory_level); + compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, + ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, + memory_level); EXPECT_EQ(0, compressor.checksum()); TestUtility::feedBufferWithRandomCharacters(buffer, 4096); @@ -139,9 +147,9 @@ TEST_F(ZlibCompressorImplTest, CallingFinishOnly) { Buffer::OwnedImpl buffer; ZlibCompressorImplTester compressor; - compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, - gzip_window_bits, memory_level); + compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, + ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, + memory_level); EXPECT_EQ(0, compressor.checksum()); TestUtility::feedBufferWithRandomCharacters(buffer, 4096); @@ -204,4 +212,7 @@ TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { } // namespace } // namespace Compressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/test/common/decompressor/BUILD b/test/extensions/compression/gzip/decompressor/BUILD similarity index 71% rename from test/common/decompressor/BUILD rename to test/extensions/compression/gzip/decompressor/BUILD index d1608797be71..4c554e1352af 100644 --- a/test/common/decompressor/BUILD +++ b/test/extensions/compression/gzip/decompressor/BUILD @@ -14,8 +14,8 @@ envoy_cc_test( deps = [ "//source/common/common:assert_lib", "//source/common/common:hex_lib", - "//source/common/compressor:compressor_lib", - "//source/common/decompressor:decompressor_lib", + "//source/extensions/compression/gzip/compressor:compressor_lib", + "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc similarity index 78% rename from test/common/decompressor/zlib_decompressor_impl_test.cc rename to test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc index 93ff4e07729d..7f08f8136ac4 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc @@ -1,13 +1,17 @@ #include "common/buffer/buffer_impl.h" #include "common/common/hex.h" -#include "common/compressor/zlib_compressor_impl.h" -#include "common/decompressor/zlib_decompressor_impl.h" + +#include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" +#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Gzip { namespace Decompressor { namespace { @@ -22,20 +26,20 @@ class ZlibDecompressorImplTest : public testing::Test { Buffer::OwnedImpl buffer; Buffer::OwnedImpl accumulation_buffer; - Envoy::Compressor::ZlibCompressorImpl compressor; + Compressor::ZlibCompressorImpl compressor; compressor.init(comp_level, comp_strategy, window_bits, memory_level); std::string original_text{}; for (uint64_t i = 0; i < 30; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(buffer.toString()); - compressor.compress(buffer, Compressor::State::Flush); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, Compressor::State::Finish); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); accumulation_buffer.add(buffer); drainBuffer(buffer); @@ -87,16 +91,16 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { Buffer::OwnedImpl compressor_buffer; Buffer::OwnedImpl decompressor_output_buffer; - Envoy::Compressor::ZlibCompressorImpl compressor; + Compressor::ZlibCompressorImpl compressor; ASSERT_EQ(0, compressor.checksum()); - compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, - gzip_window_bits, memory_level); + compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, + memory_level); ASSERT_EQ(0, compressor.checksum()); TestUtility::feedBufferWithRandomCharacters(compressor_buffer, 4096); - compressor.compress(compressor_buffer, Compressor::State::Flush); + compressor.compress(compressor_buffer, Envoy::Compression::Compressor::State::Flush); ASSERT_TRUE(compressor.checksum() > 0); ZlibDecompressorImpl decompressor; @@ -119,23 +123,23 @@ TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { Buffer::OwnedImpl accumulation_buffer; Buffer::OwnedImpl empty_buffer; - Envoy::Compressor::ZlibCompressorImpl compressor; - compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, - gzip_window_bits, memory_level); + Compressor::ZlibCompressorImpl compressor; + compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, + memory_level); std::string original_text{}; for (uint64_t i = 0; i < 20; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(buffer.toString()); - compressor.compress(buffer, Compressor::State::Flush); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, Compressor::State::Finish); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); ASSERT_GE(10, buffer.length()); accumulation_buffer.add(buffer); @@ -186,23 +190,23 @@ TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { Buffer::OwnedImpl buffer; Buffer::OwnedImpl accumulation_buffer; - Envoy::Compressor::ZlibCompressorImpl compressor; - compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, - gzip_window_bits, memory_level); + Compressor::ZlibCompressorImpl compressor; + compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, + memory_level); std::string original_text{}; for (uint64_t i = 0; i < 20; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(buffer.toString()); - compressor.compress(buffer, Compressor::State::Flush); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, Compressor::State::Finish); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); ASSERT_GE(10, buffer.length()); accumulation_buffer.add(buffer); @@ -228,19 +232,19 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { for (uint64_t i = 1; i < 10; ++i) { testcompressDecompressWithUncommonParams( Compressor::ZlibCompressorImpl::CompressionLevel::Best, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); + Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); testcompressDecompressWithUncommonParams( Compressor::ZlibCompressorImpl::CompressionLevel::Best, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); + Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); testcompressDecompressWithUncommonParams( Compressor::ZlibCompressorImpl::CompressionLevel::Speed, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, 15, i); + Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, 15, i); testcompressDecompressWithUncommonParams( Compressor::ZlibCompressorImpl::CompressionLevel::Speed, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, 15, i); + Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, 15, i); } } @@ -262,12 +266,12 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressOfMultipleSlices) { const uint64_t num_slices = buffer.getRawSlices().size(); EXPECT_EQ(num_slices, 20); - Envoy::Compressor::ZlibCompressorImpl compressor; - compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, - gzip_window_bits, memory_level); + Compressor::ZlibCompressorImpl compressor; + compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, + memory_level); - compressor.compress(buffer, Compressor::State::Flush); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); ZlibDecompressorImpl decompressor; @@ -286,4 +290,7 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressOfMultipleSlices) { } // namespace } // namespace Decompressor +} // namespace Gzip +} // namespace Compression +} // namespace Extensions } // namespace Envoy diff --git a/test/extensions/compression/gzip/gzip_compressor_library_test.cc b/test/extensions/compression/gzip/gzip_compressor_library_test.cc deleted file mode 100644 index 2e339bd7adfb..000000000000 --- a/test/extensions/compression/gzip/gzip_compressor_library_test.cc +++ /dev/null @@ -1,109 +0,0 @@ -#include "extensions/compression/gzip/compressor/config.h" - -#include "test/test_common/utility.h" - -#include "gtest/gtest.h" - -namespace Envoy { -namespace Extensions { -namespace Compression { -namespace Gzip { -namespace Compressor { - -class GzipTest : public testing::Test { -protected: - void SetUp() override { setUpGzip("{}"); } - - // GzipTest Helpers - void setUpGzip(std::string&& json) { - envoy::extensions::compression::gzip::compressor::v3::Gzip gzip; - TestUtility::loadFromJson(json, gzip); - factory_ = std::make_unique(gzip); - } - - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel compressionLevel() const { - return factory_->compression_level_; - } - - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy compressionStrategy() const { - return factory_->compression_strategy_; - } - - int32_t memoryLevel() const { return factory_->memory_level_; } - int32_t windowBits() const { return factory_->window_bits_; } - int32_t chunkSize() const { return factory_->chunk_size_; } - - void expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy strategy, - absl::string_view strategy_name, - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel level, absl::string_view level_name) { - setUpGzip(fmt::format( - R"EOF({{"compression_strategy": "{}", "compression_level": "{}", "memory_level": 6, "window_bits": 27, "chunk_size": 10000}})EOF", - strategy_name, level_name)); - EXPECT_EQ(strategy, compressionStrategy()); - EXPECT_EQ(level, compressionLevel()); - EXPECT_EQ(6, memoryLevel()); - EXPECT_EQ(27, windowBits()); - EXPECT_EQ(10000, chunkSize()); - } - - std::unique_ptr factory_; -}; - -// Default config values. -TEST_F(GzipTest, DefaultConfigValues) { - EXPECT_EQ(5, memoryLevel()); - EXPECT_EQ(28, windowBits()); - EXPECT_EQ(4096, chunkSize()); - EXPECT_EQ(Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, - compressionStrategy()); - EXPECT_EQ(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, compressionLevel()); -} - -TEST_F(GzipTest, AvailableCombinationCompressionStrategyAndLevelConfig) { - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, "FILTERED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST_COMPRESSION"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, "HUFFMAN_ONLY", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST_COMPRESSION"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, "RLE", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Speed, "BEST_SPEED"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, "DEFAULT_STRATEGY", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, "DEFAULT_COMPRESSION"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level1, "COMPRESSION_LEVEL_1"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level2, "COMPRESSION_LEVEL_2"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level3, "COMPRESSION_LEVEL_3"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level4, "COMPRESSION_LEVEL_4"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level5, "COMPRESSION_LEVEL_5"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level6, "COMPRESSION_LEVEL_6"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level7, "COMPRESSION_LEVEL_7"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level8, "COMPRESSION_LEVEL_8"); - expectValidCompressionStrategyAndLevel( - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Level9, "COMPRESSION_LEVEL_9"); -} - -} // namespace Compressor -} // namespace Gzip -} // namespace Compression -} // namespace Extensions -} // namespace Envoy diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index a55ad21ee108..dc375912e801 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -22,8 +22,8 @@ namespace HttpFilters { namespace Common { namespace Compressors { -class MockCompressor : public Compressor::Compressor { - void compress(Buffer::Instance&, ::Envoy::Compressor::State) override {} +class MockCompressor : public Envoy::Compression::Compressor::Compressor { + void compress(Buffer::Instance&, Envoy::Compression::Compressor::State) override {} }; class MockCompressorFilterConfig : public CompressorFilterConfig { @@ -35,7 +35,9 @@ class MockCompressorFilterConfig : public CompressorFilterConfig { : CompressorFilterConfig(compressor, stats_prefix + compressor_name + ".", scope, runtime, compressor_name) {} - Compressor::CompressorPtr makeCompressor() override { return std::make_unique(); } + Envoy::Compression::Compressor::CompressorPtr makeCompressor() override { + return std::make_unique(); + } }; class CompressorFilterTest : public testing::Test { diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD index d192a617454f..819d726d3840 100644 --- a/test/extensions/filters/http/compressor/BUILD +++ b/test/extensions/filters/http/compressor/BUILD @@ -11,6 +11,18 @@ load( envoy_package() +envoy_extension_cc_test( + name = "gzip_compressor_filter_test", + srcs = ["gzip_compressor_filter_test.cc"], + extension_name = "envoy.filters.http.compressor", + deps = [ + "//source/extensions/compression/gzip/compressor:config", + "//source/extensions/compression/gzip/decompressor:decompressor_lib", + "//source/extensions/filters/http/compressor:compressor_filter_lib", + "//test/mocks/http:http_mocks", + ], +) + envoy_extension_cc_test( name = "compressor_filter_integration_test", srcs = [ @@ -18,8 +30,8 @@ envoy_extension_cc_test( ], extension_name = "envoy.filters.http.compressor", deps = [ - "//source/common/decompressor:decompressor_lib", "//source/extensions/compression/gzip/compressor:config", + "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//source/extensions/filters/http/compressor:config", "//test/integration:http_integration_lib", "//test/test_common:simulated_time_system_lib", diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 92c4ca5b3657..47e0c99ef285 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -1,6 +1,6 @@ #include "envoy/event/timer.h" -#include "common/decompressor/zlib_decompressor_impl.h" +#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "test/integration/http_integration.h" #include "test/test_common/simulated_time_system.h" @@ -94,7 +94,7 @@ class CompressorIntegrationTest : public testing::TestWithParam #include "common/common/hex.h" -#include "common/decompressor/zlib_decompressor_impl.h" #include "extensions/compression/gzip/compressor/config.h" +#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "extensions/filters/http/compressor/compressor_filter.h" #include "test/mocks/http/mocks.h" @@ -122,7 +122,7 @@ class GzipCompressorFilterTest : public testing::Test { std::shared_ptr config_; std::unique_ptr filter_; Buffer::OwnedImpl data_; - Decompressor::ZlibDecompressorImpl decompressor_; + Compression::Gzip::Decompressor::ZlibDecompressorImpl decompressor_; Buffer::OwnedImpl decompressed_data_; std::string expected_str_; Stats::TestUtil::TestStore stats_; diff --git a/test/extensions/filters/http/gzip/BUILD b/test/extensions/filters/http/gzip/BUILD index 197789425917..583342ee598d 100644 --- a/test/extensions/filters/http/gzip/BUILD +++ b/test/extensions/filters/http/gzip/BUILD @@ -16,9 +16,9 @@ envoy_extension_cc_test( srcs = ["gzip_filter_test.cc"], extension_name = "envoy.filters.http.gzip", deps = [ - "//source/common/compressor:compressor_lib", - "//source/common/decompressor:decompressor_lib", "//source/common/protobuf:utility_lib", + "//source/extensions/compression/gzip/compressor:compressor_lib", + "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//source/extensions/filters/http/gzip:config", "//source/extensions/filters/http/gzip:gzip_filter_lib", "//test/mocks/http:http_mocks", @@ -37,7 +37,7 @@ envoy_extension_cc_test( ], extension_name = "envoy.filters.http.gzip", deps = [ - "//source/common/decompressor:decompressor_lib", + "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//source/extensions/filters/http/gzip:config", "//test/integration:http_integration_lib", "//test/test_common:simulated_time_system_lib", diff --git a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc index ad8c658e0502..eb3c7782925f 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc @@ -1,6 +1,6 @@ #include "envoy/event/timer.h" -#include "common/decompressor/zlib_decompressor_impl.h" +#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "test/integration/http_integration.h" #include "test/test_common/simulated_time_system.h" @@ -100,7 +100,7 @@ class GzipIntegrationTest : public testing::TestWithParamcompressionStrategy()); @@ -157,7 +159,7 @@ class GzipFilterTest : public testing::Test { std::shared_ptr config_; std::unique_ptr filter_; Buffer::OwnedImpl data_; - Decompressor::ZlibDecompressorImpl decompressor_; + Compression::Gzip::Decompressor::ZlibDecompressorImpl decompressor_; Buffer::OwnedImpl decompressed_data_; std::string expected_str_; Stats::TestUtil::TestStore stats_; @@ -192,26 +194,26 @@ TEST_F(GzipFilterTest, DefaultConfigValues) { EXPECT_EQ(28, config_->windowBits()); EXPECT_EQ(false, config_->disableOnEtagHeader()); EXPECT_EQ(false, config_->removeAcceptEncodingHeader()); - EXPECT_EQ(Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + EXPECT_EQ(Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, config_->compressionStrategy()); - EXPECT_EQ(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + EXPECT_EQ(Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, config_->compressionLevel()); EXPECT_EQ(18, config_->contentTypeValues().size()); } TEST_F(GzipFilterTest, AvailableCombinationCompressionStrategyAndLevelConfig) { expectValidCompressionStrategyAndLevel( - Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, "FILTERED", - Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST"); + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, "FILTERED", + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST"); expectValidCompressionStrategyAndLevel( - Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, "HUFFMAN", - Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST"); + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, "HUFFMAN", + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Best, "BEST"); expectValidCompressionStrategyAndLevel( - Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, "RLE", - Compressor::ZlibCompressorImpl::CompressionLevel::Speed, "SPEED"); + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, "RLE", + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Speed, "SPEED"); expectValidCompressionStrategyAndLevel( - Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, "DEFAULT", - Compressor::ZlibCompressorImpl::CompressionLevel::Standard, "DEFAULT"); + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, "DEFAULT", + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, "DEFAULT"); } // Acceptance Testing with default configuration. From d833c1ae92def4bc7cf7bc82dc4afa52819d4f32 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 20 Apr 2020 15:43:56 +0300 Subject: [PATCH 27/56] run check_format.py fix Signed-off-by: Dmitry Rozhkov --- docs/root/version_history/current.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 0c97aa2d5bdb..be7dca0aaa22 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -5,8 +5,8 @@ Changes ------- * access loggers: added GRPC_STATUS operator on logging format. -* compressor: generic :ref:`compressor ` filter exposed to users. * access loggers: extened specifier for FilterStateFormatter to output :ref:`unstructured log string `. +* compressor: generic :ref:`compressor ` filter exposed to users. * dynamic forward proxy: added :ref:`SNI based dynamic forward proxy ` support. * fault: added support for controlling the percentage of requests that abort, delay and response rate limits faults are applied to using :ref:`HTTP headers ` to the HTTP fault filter. From 388d8bc0cc20c7a28dcc7c6730e0f3fde154a8c7 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 20 Apr 2020 16:13:17 +0300 Subject: [PATCH 28/56] silence clang-tidy warnings Signed-off-by: Dmitry Rozhkov --- source/extensions/filters/http/gzip/gzip_filter.cc | 13 +++++++++++++ tools/spelling/spelling_dictionary.txt | 1 + 2 files changed, 14 insertions(+) diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index a7d9cb0bc14b..2610abfe0da6 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -83,6 +83,10 @@ GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip: // what is accepted by the generic compressor. if (gzip.has_compressor()) { compressor.set_allocated_content_length( + // According to + // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage + // the message Compressor takes ownership of the allocated Protobuf::Uint32Value object. + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) new Protobuf::UInt32Value(gzip.compressor().content_length())); for (const std::string& ctype : gzip.compressor().content_type()) { compressor.add_content_type(ctype); @@ -93,6 +97,11 @@ GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip: auto feature_flag(new envoy::config::core::v3::RuntimeFeatureFlag()); feature_flag->set_runtime_key(gzip.compressor().runtime_enabled().runtime_key()); feature_flag->set_allocated_default_value( + // According to + // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage + // the message RuntimeFeatureFlag takes ownership of the allocated Protobuf::BoolValue + // object. + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) new Protobuf::BoolValue(gzip.compressor().runtime_enabled().default_value())); compressor.set_allocated_runtime_enabled(feature_flag); } @@ -101,6 +110,10 @@ GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip: if (gzip.has_hidden_envoy_deprecated_content_length()) { compressor.set_allocated_content_length( + // According to + // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage + // the message Compressor takes ownership of the allocated Protobuf::Uint32Value object. + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) new Protobuf::UInt32Value(gzip.hidden_envoy_deprecated_content_length())); } for (const std::string& ctype : gzip.hidden_envoy_deprecated_content_type()) { diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 21de3bb52722..a37d8678c998 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -471,6 +471,7 @@ coroutines cors cout coverity +cplusplus cpuset creds crypto From 6ea514a13fd0d5424aaed45fb4dd4478ee830506 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 21 Apr 2020 15:58:36 +0300 Subject: [PATCH 29/56] revert moving decompressor code to reduce patch size Signed-off-by: Dmitry Rozhkov --- .../{compression => }/decompressor/BUILD | 0 .../decompressor/decompressor.h | 2 - source/common/common/BUILD | 10 +++ .../common => common/common/zlib}/base.cc | 12 +-- .../gzip/common => common/common/zlib}/base.h | 10 +-- .../gzip => common}/decompressor/BUILD | 4 +- .../decompressor/zlib_decompressor_impl.cc | 10 +-- .../decompressor/zlib_decompressor_impl.h | 15 +--- .../extensions/compression/gzip/common/BUILD | 19 ---- .../compression/gzip/compressor/BUILD | 2 +- .../gzip/compressor/zlib_compressor_impl.cc | 2 +- .../gzip/compressor/zlib_compressor_impl.h | 4 +- .../gzip => common}/decompressor/BUILD | 2 +- .../zlib_decompressor_impl_test.cc | 90 ++++++++++--------- test/common/stats/BUILD | 2 +- .../compression/gzip/compressor/BUILD | 2 +- .../gzip/compressor/compressor_fuzz_test.cc | 2 +- test/extensions/filters/http/compressor/BUILD | 4 +- .../compressor_filter_integration_test.cc | 4 +- .../compressor/gzip_compressor_filter_test.cc | 4 +- test/extensions/filters/http/gzip/BUILD | 4 +- .../http/gzip/gzip_filter_integration_test.cc | 4 +- .../filters/http/gzip/gzip_filter_test.cc | 4 +- 23 files changed, 90 insertions(+), 122 deletions(-) rename include/envoy/{compression => }/decompressor/BUILD (100%) rename include/envoy/{compression => }/decompressor/decompressor.h (92%) rename source/{extensions/compression/gzip/common => common/common/zlib}/base.cc (73%) rename source/{extensions/compression/gzip/common => common/common/zlib}/base.h (84%) rename source/{extensions/compression/gzip => common}/decompressor/BUILD (76%) rename source/{extensions/compression/gzip => common}/decompressor/zlib_decompressor_impl.cc (89%) rename source/{extensions/compression/gzip => common}/decompressor/zlib_decompressor_impl.h (81%) delete mode 100644 source/extensions/compression/gzip/common/BUILD rename test/{extensions/compression/gzip => common}/decompressor/BUILD (85%) rename test/{extensions/compression/gzip => common}/decompressor/zlib_decompressor_impl_test.cc (71%) diff --git a/include/envoy/compression/decompressor/BUILD b/include/envoy/decompressor/BUILD similarity index 100% rename from include/envoy/compression/decompressor/BUILD rename to include/envoy/decompressor/BUILD diff --git a/include/envoy/compression/decompressor/decompressor.h b/include/envoy/decompressor/decompressor.h similarity index 92% rename from include/envoy/compression/decompressor/decompressor.h rename to include/envoy/decompressor/decompressor.h index 154b0aaa742a..d694aa50ca1c 100644 --- a/include/envoy/compression/decompressor/decompressor.h +++ b/include/envoy/decompressor/decompressor.h @@ -3,7 +3,6 @@ #include "envoy/buffer/buffer.h" namespace Envoy { -namespace Compression { namespace Decompressor { /** @@ -23,5 +22,4 @@ class Decompressor { }; } // namespace Decompressor -} // namespace Compression } // namespace Envoy diff --git a/source/common/common/BUILD b/source/common/common/BUILD index ccc60a82c2fc..af25042d18c5 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -420,6 +420,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "zlib_base_lib", + srcs = ["zlib/base.cc"], + hdrs = ["zlib/base.h"], + external_deps = ["zlib"], + deps = [ + "//source/common/buffer:buffer_lib", + ], +) + envoy_cc_library( name = "statusor_lib", hdrs = ["statusor.h"], diff --git a/source/extensions/compression/gzip/common/base.cc b/source/common/common/zlib/base.cc similarity index 73% rename from source/extensions/compression/gzip/common/base.cc rename to source/common/common/zlib/base.cc index 1bd594932024..5336f35f8735 100644 --- a/source/extensions/compression/gzip/common/base.cc +++ b/source/common/common/zlib/base.cc @@ -1,10 +1,7 @@ -#include "extensions/compression/gzip/common/base.h" +#include "common/common/zlib/base.h" namespace Envoy { -namespace Extensions { -namespace Compression { -namespace Gzip { -namespace Common { +namespace Zlib { Base::Base(uint64_t chunk_size, std::function zstream_deleter) : chunk_size_{chunk_size}, chunk_char_ptr_(new unsigned char[chunk_size]), @@ -23,8 +20,5 @@ void Base::updateOutput(Buffer::Instance& output_buffer) { zstream_ptr_->next_out = chunk_char_ptr_.get(); } -} // namespace Common -} // namespace Gzip -} // namespace Compression -} // namespace Extensions +} // namespace Zlib } // namespace Envoy diff --git a/source/extensions/compression/gzip/common/base.h b/source/common/common/zlib/base.h similarity index 84% rename from source/extensions/compression/gzip/common/base.h rename to source/common/common/zlib/base.h index 720ebc68f118..4f427fb90985 100644 --- a/source/extensions/compression/gzip/common/base.h +++ b/source/common/common/zlib/base.h @@ -7,10 +7,7 @@ #include "zlib.h" namespace Envoy { -namespace Extensions { -namespace Compression { -namespace Gzip { -namespace Common { +namespace Zlib { /** * Shared code between the compressor and the decompressor. @@ -39,8 +36,5 @@ class Base { const std::unique_ptr> zstream_ptr_; }; -} // namespace Common -} // namespace Gzip -} // namespace Compression -} // namespace Extensions +} // namespace Zlib } // namespace Envoy diff --git a/source/extensions/compression/gzip/decompressor/BUILD b/source/common/decompressor/BUILD similarity index 76% rename from source/extensions/compression/gzip/decompressor/BUILD rename to source/common/decompressor/BUILD index 323930ba9a7a..dfdf8f9b90ed 100644 --- a/source/extensions/compression/gzip/decompressor/BUILD +++ b/source/common/decompressor/BUILD @@ -14,10 +14,10 @@ envoy_cc_library( hdrs = ["zlib_decompressor_impl.h"], external_deps = ["zlib"], deps = [ - "//include/envoy/compression/decompressor:decompressor_interface", + "//include/envoy/decompressor:decompressor_interface", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", - "//source/extensions/compression/gzip/common:gzip_base_lib", + "//source/common/common:zlib_base_lib", ], ) diff --git a/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc b/source/common/decompressor/zlib_decompressor_impl.cc similarity index 89% rename from source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc rename to source/common/decompressor/zlib_decompressor_impl.cc index 42e7d12e1137..55dffc6d3609 100644 --- a/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc +++ b/source/common/decompressor/zlib_decompressor_impl.cc @@ -1,4 +1,4 @@ -#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" +#include "common/decompressor/zlib_decompressor_impl.h" #include @@ -9,15 +9,12 @@ #include "absl/container/fixed_array.h" namespace Envoy { -namespace Extensions { -namespace Compression { -namespace Gzip { namespace Decompressor { ZlibDecompressorImpl::ZlibDecompressorImpl() : ZlibDecompressorImpl(4096) {} ZlibDecompressorImpl::ZlibDecompressorImpl(uint64_t chunk_size) - : Common::Base(chunk_size, [](z_stream* z) { + : Zlib::Base(chunk_size, [](z_stream* z) { inflateEnd(z); delete z; }) { @@ -78,7 +75,4 @@ bool ZlibDecompressorImpl::inflateNext() { } } // namespace Decompressor -} // namespace Gzip -} // namespace Compression -} // namespace Extensions } // namespace Envoy diff --git a/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.h b/source/common/decompressor/zlib_decompressor_impl.h similarity index 81% rename from source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.h rename to source/common/decompressor/zlib_decompressor_impl.h index b290fb851915..8d5627fc6c31 100644 --- a/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.h +++ b/source/common/decompressor/zlib_decompressor_impl.h @@ -1,24 +1,20 @@ #pragma once -#include "envoy/compression/decompressor/decompressor.h" +#include "envoy/decompressor/decompressor.h" #include "common/common/logger.h" - -#include "extensions/compression/gzip/common/base.h" +#include "common/common/zlib/base.h" #include "zlib.h" namespace Envoy { -namespace Extensions { -namespace Compression { -namespace Gzip { namespace Decompressor { /** * Implementation of decompressor's interface. */ -class ZlibDecompressorImpl : public Common::Base, - public Envoy::Compression::Decompressor::Decompressor, +class ZlibDecompressorImpl : public Zlib::Base, + public Decompressor, public Logger::Loggable { public: ZlibDecompressorImpl(); @@ -53,7 +49,4 @@ class ZlibDecompressorImpl : public Common::Base, }; } // namespace Decompressor -} // namespace Gzip -} // namespace Compression -} // namespace Extensions } // namespace Envoy diff --git a/source/extensions/compression/gzip/common/BUILD b/source/extensions/compression/gzip/common/BUILD deleted file mode 100644 index 7c9583e7985f..000000000000 --- a/source/extensions/compression/gzip/common/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load( - "//bazel:envoy_build_system.bzl", - "envoy_cc_library", - "envoy_package", -) - -envoy_package() - -envoy_cc_library( - name = "gzip_base_lib", - srcs = ["base.cc"], - hdrs = ["base.h"], - external_deps = ["zlib"], - deps = [ - "//source/common/buffer:buffer_lib", - ], -) diff --git a/source/extensions/compression/gzip/compressor/BUILD b/source/extensions/compression/gzip/compressor/BUILD index 54f476883068..f7c1d5f51b1d 100644 --- a/source/extensions/compression/gzip/compressor/BUILD +++ b/source/extensions/compression/gzip/compressor/BUILD @@ -18,7 +18,7 @@ envoy_cc_library( "//include/envoy/compression/compressor:compressor_interface", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", - "//source/extensions/compression/gzip/common:gzip_base_lib", + "//source/common/common:zlib_base_lib", ], ) diff --git a/source/extensions/compression/gzip/compressor/zlib_compressor_impl.cc b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.cc index 39f5b3e5d729..432c36e97015 100644 --- a/source/extensions/compression/gzip/compressor/zlib_compressor_impl.cc +++ b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.cc @@ -17,7 +17,7 @@ namespace Compressor { ZlibCompressorImpl::ZlibCompressorImpl() : ZlibCompressorImpl(4096) {} ZlibCompressorImpl::ZlibCompressorImpl(uint64_t chunk_size) - : Common::Base(chunk_size, [](z_stream* z) { + : Zlib::Base(chunk_size, [](z_stream* z) { deflateEnd(z); delete z; }) { diff --git a/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h index f182a6ff9d40..d49901e434d9 100644 --- a/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h +++ b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h @@ -2,7 +2,7 @@ #include "envoy/compression/compressor/compressor.h" -#include "extensions/compression/gzip/common/base.h" +#include "common/common/zlib/base.h" #include "zlib.h" @@ -15,7 +15,7 @@ namespace Compressor { /** * Implementation of compressor's interface. */ -class ZlibCompressorImpl : public Common::Base, public Envoy::Compression::Compressor::Compressor { +class ZlibCompressorImpl : public Zlib::Base, public Envoy::Compression::Compressor::Compressor { public: ZlibCompressorImpl(); diff --git a/test/extensions/compression/gzip/decompressor/BUILD b/test/common/decompressor/BUILD similarity index 85% rename from test/extensions/compression/gzip/decompressor/BUILD rename to test/common/decompressor/BUILD index 4c554e1352af..55703c5b25ab 100644 --- a/test/extensions/compression/gzip/decompressor/BUILD +++ b/test/common/decompressor/BUILD @@ -14,8 +14,8 @@ envoy_cc_test( deps = [ "//source/common/common:assert_lib", "//source/common/common:hex_lib", + "//source/common/decompressor:decompressor_lib", "//source/extensions/compression/gzip/compressor:compressor_lib", - "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc similarity index 71% rename from test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc rename to test/common/decompressor/zlib_decompressor_impl_test.cc index 7f08f8136ac4..99e0b43e83f6 100644 --- a/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -1,17 +1,14 @@ #include "common/buffer/buffer_impl.h" #include "common/common/hex.h" +#include "common/decompressor/zlib_decompressor_impl.h" #include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" -#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" namespace Envoy { -namespace Extensions { -namespace Compression { -namespace Gzip { namespace Decompressor { namespace { @@ -20,26 +17,27 @@ class ZlibDecompressorImplTest : public testing::Test { void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } void testcompressDecompressWithUncommonParams( - Compressor::ZlibCompressorImpl::CompressionLevel comp_level, - Compressor::ZlibCompressorImpl::CompressionStrategy comp_strategy, int64_t window_bits, - uint64_t memory_level) { + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel comp_level, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy + comp_strategy, + int64_t window_bits, uint64_t memory_level) { Buffer::OwnedImpl buffer; Buffer::OwnedImpl accumulation_buffer; - Compressor::ZlibCompressorImpl compressor; + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl compressor; compressor.init(comp_level, comp_strategy, window_bits, memory_level); std::string original_text{}; for (uint64_t i = 0; i < 30; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(buffer.toString()); - compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + compressor.compress(buffer, Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + compressor.compress(buffer, Compression::Compressor::State::Finish); accumulation_buffer.add(buffer); drainBuffer(buffer); @@ -91,16 +89,17 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { Buffer::OwnedImpl compressor_buffer; Buffer::OwnedImpl decompressor_output_buffer; - Compressor::ZlibCompressorImpl compressor; + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl compressor; ASSERT_EQ(0, compressor.checksum()); - compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); + compressor.init( + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + gzip_window_bits, memory_level); ASSERT_EQ(0, compressor.checksum()); TestUtility::feedBufferWithRandomCharacters(compressor_buffer, 4096); - compressor.compress(compressor_buffer, Envoy::Compression::Compressor::State::Flush); + compressor.compress(compressor_buffer, Compression::Compressor::State::Flush); ASSERT_TRUE(compressor.checksum() > 0); ZlibDecompressorImpl decompressor; @@ -123,23 +122,24 @@ TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { Buffer::OwnedImpl accumulation_buffer; Buffer::OwnedImpl empty_buffer; - Compressor::ZlibCompressorImpl compressor; - compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl compressor; + compressor.init( + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + gzip_window_bits, memory_level); std::string original_text{}; for (uint64_t i = 0; i < 20; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(buffer.toString()); - compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + compressor.compress(buffer, Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + compressor.compress(buffer, Compression::Compressor::State::Finish); ASSERT_GE(10, buffer.length()); accumulation_buffer.add(buffer); @@ -190,23 +190,24 @@ TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { Buffer::OwnedImpl buffer; Buffer::OwnedImpl accumulation_buffer; - Compressor::ZlibCompressorImpl compressor; - compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); + Envoy::Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl compressor; + compressor.init( + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + gzip_window_bits, memory_level); std::string original_text{}; for (uint64_t i = 0; i < 20; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(buffer.toString()); - compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + compressor.compress(buffer, Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + compressor.compress(buffer, Compression::Compressor::State::Finish); ASSERT_GE(10, buffer.length()); accumulation_buffer.add(buffer); @@ -231,20 +232,25 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { // Test with different memory levels. for (uint64_t i = 1; i < 10; ++i) { testcompressDecompressWithUncommonParams( - Compressor::ZlibCompressorImpl::CompressionLevel::Best, - Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Best, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, + i); testcompressDecompressWithUncommonParams( - Compressor::ZlibCompressorImpl::CompressionLevel::Best, - Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Best, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, + i); testcompressDecompressWithUncommonParams( - Compressor::ZlibCompressorImpl::CompressionLevel::Speed, - Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, 15, i); + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Speed, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, + 15, i); testcompressDecompressWithUncommonParams( - Compressor::ZlibCompressorImpl::CompressionLevel::Speed, - Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, 15, i); + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Speed, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy:: + Filtered, + 15, i); } } @@ -266,12 +272,13 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressOfMultipleSlices) { const uint64_t num_slices = buffer.getRawSlices().size(); EXPECT_EQ(num_slices, 20); - Compressor::ZlibCompressorImpl compressor; - compressor.init(Compressor::ZlibCompressorImpl::CompressionLevel::Standard, - Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl compressor; + compressor.init( + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + gzip_window_bits, memory_level); - compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + compressor.compress(buffer, Compression::Compressor::State::Flush); accumulation_buffer.add(buffer); ZlibDecompressorImpl decompressor; @@ -290,7 +297,4 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressOfMultipleSlices) { } // namespace } // namespace Decompressor -} // namespace Gzip -} // namespace Compression -} // namespace Extensions } // namespace Envoy diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index f7e99eaf9ffe..ad8e7885cd1c 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -157,8 +157,8 @@ envoy_cc_fuzz_test( ":stat_test_utility_lib", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", + "//source/common/decompressor:decompressor_lib", "//source/common/stats:symbol_table_lib", - "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//test/fuzz:utility_lib", ], ) diff --git a/test/extensions/compression/gzip/compressor/BUILD b/test/extensions/compression/gzip/compressor/BUILD index 874f4f869dcd..bf8f0b3fdf1f 100644 --- a/test/extensions/compression/gzip/compressor/BUILD +++ b/test/extensions/compression/gzip/compressor/BUILD @@ -20,8 +20,8 @@ envoy_cc_fuzz_test( deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", + "//source/common/decompressor:decompressor_lib", "//source/extensions/compression/gzip/compressor:compressor_lib", - "//source/extensions/compression/gzip/decompressor:decompressor_lib", ], ) diff --git a/test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc b/test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc index da76007c8989..b9a194c0c15c 100644 --- a/test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc +++ b/test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc @@ -1,8 +1,8 @@ #include "common/buffer/buffer_impl.h" #include "common/common/assert.h" +#include "common/decompressor/zlib_decompressor_impl.h" #include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" -#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "test/fuzz/fuzz_runner.h" diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD index 819d726d3840..6a9f3808f52b 100644 --- a/test/extensions/filters/http/compressor/BUILD +++ b/test/extensions/filters/http/compressor/BUILD @@ -16,8 +16,8 @@ envoy_extension_cc_test( srcs = ["gzip_compressor_filter_test.cc"], extension_name = "envoy.filters.http.compressor", deps = [ + "//source/common/decompressor:decompressor_lib", "//source/extensions/compression/gzip/compressor:config", - "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//source/extensions/filters/http/compressor:compressor_filter_lib", "//test/mocks/http:http_mocks", ], @@ -30,8 +30,8 @@ envoy_extension_cc_test( ], extension_name = "envoy.filters.http.compressor", deps = [ + "//source/common/decompressor:decompressor_lib", "//source/extensions/compression/gzip/compressor:config", - "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//source/extensions/filters/http/compressor:config", "//test/integration:http_integration_lib", "//test/test_common:simulated_time_system_lib", diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 47e0c99ef285..92c4ca5b3657 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -1,6 +1,6 @@ #include "envoy/event/timer.h" -#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" +#include "common/decompressor/zlib_decompressor_impl.h" #include "test/integration/http_integration.h" #include "test/test_common/simulated_time_system.h" @@ -94,7 +94,7 @@ class CompressorIntegrationTest : public testing::TestWithParam #include "common/common/hex.h" +#include "common/decompressor/zlib_decompressor_impl.h" #include "extensions/compression/gzip/compressor/config.h" -#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" #include "extensions/filters/http/compressor/compressor_filter.h" #include "test/mocks/http/mocks.h" @@ -122,7 +122,7 @@ class GzipCompressorFilterTest : public testing::Test { std::shared_ptr config_; std::unique_ptr filter_; Buffer::OwnedImpl data_; - Compression::Gzip::Decompressor::ZlibDecompressorImpl decompressor_; + Decompressor::ZlibDecompressorImpl decompressor_; Buffer::OwnedImpl decompressed_data_; std::string expected_str_; Stats::TestUtil::TestStore stats_; diff --git a/test/extensions/filters/http/gzip/BUILD b/test/extensions/filters/http/gzip/BUILD index 583342ee598d..d45f94e1bbd3 100644 --- a/test/extensions/filters/http/gzip/BUILD +++ b/test/extensions/filters/http/gzip/BUILD @@ -16,9 +16,9 @@ envoy_extension_cc_test( srcs = ["gzip_filter_test.cc"], extension_name = "envoy.filters.http.gzip", deps = [ + "//source/common/decompressor:decompressor_lib", "//source/common/protobuf:utility_lib", "//source/extensions/compression/gzip/compressor:compressor_lib", - "//source/extensions/compression/gzip/decompressor:decompressor_lib", "//source/extensions/filters/http/gzip:config", "//source/extensions/filters/http/gzip:gzip_filter_lib", "//test/mocks/http:http_mocks", @@ -37,7 +37,7 @@ envoy_extension_cc_test( ], extension_name = "envoy.filters.http.gzip", deps = [ - "//source/extensions/compression/gzip/decompressor:decompressor_lib", + "//source/common/decompressor:decompressor_lib", "//source/extensions/filters/http/gzip:config", "//test/integration:http_integration_lib", "//test/test_common:simulated_time_system_lib", diff --git a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc index eb3c7782925f..ad8c658e0502 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_integration_test.cc @@ -1,6 +1,6 @@ #include "envoy/event/timer.h" -#include "extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" +#include "common/decompressor/zlib_decompressor_impl.h" #include "test/integration/http_integration.h" #include "test/test_common/simulated_time_system.h" @@ -100,7 +100,7 @@ class GzipIntegrationTest : public testing::TestWithParam config_; std::unique_ptr filter_; Buffer::OwnedImpl data_; - Compression::Gzip::Decompressor::ZlibDecompressorImpl decompressor_; + Decompressor::ZlibDecompressorImpl decompressor_; Buffer::OwnedImpl decompressed_data_; std::string expected_str_; Stats::TestUtil::TestStore stats_; From d43fb14cdb1dcbd039bc49f15f063c0df30edbce Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 21 Apr 2020 16:04:52 +0300 Subject: [PATCH 30/56] add granularity to CODEOWNERS Signed-off-by: Dmitry Rozhkov --- CODEOWNERS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 8540598b74df..edb04e0ea154 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -7,7 +7,8 @@ # access loggers /*/extensions/access_loggers/common @auni53 @zuercher # compression extensions -/*/extensions/compression/ @rojkov @junr03 +/*/extensions/compression/common/compressor @rojkov @junr03 +/*/extensions/compression/gzip/compressor @rojkov @junr03 # csrf extension /*/extensions/filters/http/csrf @dschaller @mattklein123 # original_src http filter extension From 1498c209f7a2403b024bec1b9afa87a9bd94c4d2 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 21 Apr 2020 16:56:51 +0300 Subject: [PATCH 31/56] address review comments Signed-off-by: Dmitry Rozhkov --- include/envoy/compression/compressor/BUILD | 2 +- source/extensions/compression/common/compressor/BUILD | 2 +- source/extensions/compression/gzip/compressor/config.h | 4 ++-- source/extensions/filters/http/compressor/BUILD | 2 +- .../http/compressor/gzip_compressor_filter_test.cc | 10 ++++++---- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/include/envoy/compression/compressor/BUILD b/include/envoy/compression/compressor/BUILD index 443ab7ea3c77..f9e90c9ec612 100644 --- a/include/envoy/compression/compressor/BUILD +++ b/include/envoy/compression/compressor/BUILD @@ -25,7 +25,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "compressor_library_config_interface", + name = "compressor_config_interface", hdrs = ["config.h"], deps = [ ":compressor_factory_interface", diff --git a/source/extensions/compression/common/compressor/BUILD b/source/extensions/compression/common/compressor/BUILD index cd4d685b48aa..eb16810cc8b9 100644 --- a/source/extensions/compression/common/compressor/BUILD +++ b/source/extensions/compression/common/compressor/BUILD @@ -12,8 +12,8 @@ envoy_cc_library( name = "compressor_factory_base_lib", hdrs = ["factory_base.h"], deps = [ + "//include/envoy/compression/compressor:compressor_config_interface", "//include/envoy/compression/compressor:compressor_factory_interface", - "//include/envoy/compression/compressor:compressor_library_config_interface", "//include/envoy/server:filter_config_interface", ], ) diff --git a/source/extensions/compression/gzip/compressor/config.h b/source/extensions/compression/gzip/compressor/config.h index e5d8325a3b8f..ee18aca93c7a 100644 --- a/source/extensions/compression/gzip/compressor/config.h +++ b/source/extensions/compression/gzip/compressor/config.h @@ -18,7 +18,7 @@ namespace Compressor { namespace { -const std::string& gzipStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "gzip."); } +const std::string& gzipStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "gzip.compressor."); } const std::string& gzipExtensionName() { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.compression.gzip.compressor"); } @@ -29,6 +29,7 @@ class GzipCompressorFactory : public Envoy::Compression::Compressor::CompressorF public: GzipCompressorFactory(const envoy::extensions::compression::gzip::compressor::v3::Gzip& gzip); + // Envoy::Compression::Compressor::CompressorFactory Envoy::Compression::Compressor::CompressorPtr createCompressor() override; const std::string& statsPrefix() const override { return gzipStatsPrefix(); } const std::string& contentEncoding() const override { @@ -47,7 +48,6 @@ class GzipCompressorFactory : public Envoy::Compression::Compressor::CompressorF ZlibCompressorImpl::CompressionLevel compression_level_; ZlibCompressorImpl::CompressionStrategy compression_strategy_; - const int32_t memory_level_; const int32_t window_bits_; const uint32_t chunk_size_; diff --git a/source/extensions/filters/http/compressor/BUILD b/source/extensions/filters/http/compressor/BUILD index dae412be9ca0..188ee5e38be0 100644 --- a/source/extensions/filters/http/compressor/BUILD +++ b/source/extensions/filters/http/compressor/BUILD @@ -30,7 +30,7 @@ envoy_cc_extension( security_posture = "robust_to_untrusted_downstream", deps = [ ":compressor_filter_lib", - "//include/envoy/compression/compressor:compressor_library_config_interface", + "//include/envoy/compression/compressor:compressor_config_interface", "//source/common/config:utility_lib", "//source/extensions/filters/http:well_known_names", "//source/extensions/filters/http/common:factory_base_lib", diff --git a/test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc b/test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc index d3416896af07..423770111aca 100644 --- a/test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc +++ b/test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc @@ -37,8 +37,10 @@ class GzipCompressorFilterTest : public testing::Test { const std::string uncompressed_str{decompressed_data_.toString()}; ASSERT_EQ(expected_str_.length(), uncompressed_str.length()); EXPECT_EQ(expected_str_, uncompressed_str); - EXPECT_EQ(expected_str_.length(), stats_.counter("test.gzip.total_uncompressed_bytes").value()); - EXPECT_EQ(data_.length(), stats_.counter("test.gzip.total_compressed_bytes").value()); + EXPECT_EQ(expected_str_.length(), + stats_.counter("test.gzip.compressor.total_uncompressed_bytes").value()); + EXPECT_EQ(data_.length(), + stats_.counter("test.gzip.compressor.total_compressed_bytes").value()); } void feedBuffer(uint64_t size) { @@ -72,7 +74,7 @@ class GzipCompressorFilterTest : public testing::Test { } verifyCompressedData(content_length); drainBuffer(); - EXPECT_EQ(1U, stats_.counter("test.gzip.compressed").value()); + EXPECT_EQ(1U, stats_.counter("test.gzip.compressor.compressed").value()); } void expectValidFinishedBuffer(const uint32_t content_length) { @@ -116,7 +118,7 @@ class GzipCompressorFilterTest : public testing::Test { EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, false)); Http::TestResponseTrailerMapImpl trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(trailers)); - EXPECT_EQ(1, stats_.counter("test.gzip.not_compressed").value()); + EXPECT_EQ(1, stats_.counter("test.gzip.compressor.not_compressed").value()); } std::shared_ptr config_; From 4d09cd2c8003ba32683974988137793ce3e82a75 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 21 Apr 2020 17:15:52 +0300 Subject: [PATCH 32/56] move gzip fuzzer test one level up Signed-off-by: Dmitry Rozhkov --- test/extensions/compression/gzip/BUILD | 21 ++++++++++++++++++ .../compression/gzip/compressor/BUILD | 13 ----------- ...ized-compressor_fuzz_test-5149986500640768 | Bin ...ized-compressor_fuzz_test-5407695477932032 | Bin ...ized-compressor_fuzz_test-5644831560302592 | Bin ...ized-compressor_fuzz_test-6005942746873856 | Bin .../{compressor => }/compressor_corpus/empty | 0 .../{compressor => }/compressor_corpus/noise | Bin .../{compressor => }/compressor_corpus/simple | 0 .../{compressor => }/compressor_fuzz_test.cc | 0 10 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 test/extensions/compression/gzip/BUILD rename test/extensions/compression/gzip/{compressor => }/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 (100%) rename test/extensions/compression/gzip/{compressor => }/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 (100%) rename test/extensions/compression/gzip/{compressor => }/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 (100%) rename test/extensions/compression/gzip/{compressor => }/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 (100%) rename test/extensions/compression/gzip/{compressor => }/compressor_corpus/empty (100%) rename test/extensions/compression/gzip/{compressor => }/compressor_corpus/noise (100%) rename test/extensions/compression/gzip/{compressor => }/compressor_corpus/simple (100%) rename test/extensions/compression/gzip/{compressor => }/compressor_fuzz_test.cc (100%) diff --git a/test/extensions/compression/gzip/BUILD b/test/extensions/compression/gzip/BUILD new file mode 100644 index 000000000000..98c9de21486e --- /dev/null +++ b/test/extensions/compression/gzip/BUILD @@ -0,0 +1,21 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_fuzz_test", + "envoy_package", +) + +envoy_package() + +envoy_cc_fuzz_test( + name = "compressor_fuzz_test", + srcs = ["compressor_fuzz_test.cc"], + corpus = "compressor_corpus", + deps = [ + "//source/common/buffer:buffer_lib", + "//source/common/common:assert_lib", + "//source/common/decompressor:decompressor_lib", + "//source/extensions/compression/gzip/compressor:compressor_lib", + ], +) diff --git a/test/extensions/compression/gzip/compressor/BUILD b/test/extensions/compression/gzip/compressor/BUILD index bf8f0b3fdf1f..523ccb269899 100644 --- a/test/extensions/compression/gzip/compressor/BUILD +++ b/test/extensions/compression/gzip/compressor/BUILD @@ -2,7 +2,6 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", - "envoy_cc_fuzz_test", "envoy_cc_test", "envoy_package", ) @@ -13,18 +12,6 @@ load( envoy_package() -envoy_cc_fuzz_test( - name = "compressor_fuzz_test", - srcs = ["compressor_fuzz_test.cc"], - corpus = "compressor_corpus", - deps = [ - "//source/common/buffer:buffer_lib", - "//source/common/common:assert_lib", - "//source/common/decompressor:decompressor_lib", - "//source/extensions/compression/gzip/compressor:compressor_lib", - ], -) - envoy_cc_test( name = "compressor_test", srcs = ["zlib_compressor_impl_test.cc"], diff --git a/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 b/test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 rename to test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5149986500640768 diff --git a/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 b/test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 rename to test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5407695477932032 diff --git a/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 b/test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 rename to test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-5644831560302592 diff --git a/test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 b/test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 rename to test/extensions/compression/gzip/compressor_corpus/clusterfuzz-testcase-minimized-compressor_fuzz_test-6005942746873856 diff --git a/test/extensions/compression/gzip/compressor/compressor_corpus/empty b/test/extensions/compression/gzip/compressor_corpus/empty similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_corpus/empty rename to test/extensions/compression/gzip/compressor_corpus/empty diff --git a/test/extensions/compression/gzip/compressor/compressor_corpus/noise b/test/extensions/compression/gzip/compressor_corpus/noise similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_corpus/noise rename to test/extensions/compression/gzip/compressor_corpus/noise diff --git a/test/extensions/compression/gzip/compressor/compressor_corpus/simple b/test/extensions/compression/gzip/compressor_corpus/simple similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_corpus/simple rename to test/extensions/compression/gzip/compressor_corpus/simple diff --git a/test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc b/test/extensions/compression/gzip/compressor_fuzz_test.cc similarity index 100% rename from test/extensions/compression/gzip/compressor/compressor_fuzz_test.cc rename to test/extensions/compression/gzip/compressor_fuzz_test.cc From eabd93db43c43837f05191f9295de291e596aca6 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 21 Apr 2020 17:46:09 +0300 Subject: [PATCH 33/56] adjust silencing clang-tidy warnings Signed-off-by: Dmitry Rozhkov --- source/extensions/filters/http/gzip/gzip_filter.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index 2610abfe0da6..719c5fb88caa 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -86,8 +86,8 @@ GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip: // According to // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage // the message Compressor takes ownership of the allocated Protobuf::Uint32Value object. - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) new Protobuf::UInt32Value(gzip.compressor().content_length())); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) for (const std::string& ctype : gzip.compressor().content_type()) { compressor.add_content_type(ctype); } @@ -101,8 +101,8 @@ GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip: // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage // the message RuntimeFeatureFlag takes ownership of the allocated Protobuf::BoolValue // object. - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) new Protobuf::BoolValue(gzip.compressor().runtime_enabled().default_value())); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) compressor.set_allocated_runtime_enabled(feature_flag); } return compressor; @@ -113,9 +113,9 @@ GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip: // According to // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage // the message Compressor takes ownership of the allocated Protobuf::Uint32Value object. - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) new Protobuf::UInt32Value(gzip.hidden_envoy_deprecated_content_length())); } + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) for (const std::string& ctype : gzip.hidden_envoy_deprecated_content_type()) { compressor.add_content_type(ctype); } From c69e0bb500d5452c60b7f84dd36364ba104fce58 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 22 Apr 2020 13:22:28 +0300 Subject: [PATCH 34/56] compression: rework tests Signed-off-by: Dmitry Rozhkov --- .../compression/gzip/compressor/BUILD | 14 +- .../gzip_compressor_library_test.cc | 107 ------------- .../compressor/zlib_compressor_impl_test.cc | 144 ++++++++++++------ 3 files changed, 99 insertions(+), 166 deletions(-) delete mode 100644 test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc diff --git a/test/extensions/compression/gzip/compressor/BUILD b/test/extensions/compression/gzip/compressor/BUILD index 523ccb269899..0121199e1060 100644 --- a/test/extensions/compression/gzip/compressor/BUILD +++ b/test/extensions/compression/gzip/compressor/BUILD @@ -2,7 +2,6 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", - "envoy_cc_test", "envoy_package", ) load( @@ -12,22 +11,13 @@ load( envoy_package() -envoy_cc_test( +envoy_extension_cc_test( name = "compressor_test", srcs = ["zlib_compressor_impl_test.cc"], + extension_name = "envoy.compression.gzip.compressor", deps = [ "//source/common/common:assert_lib", "//source/common/common:hex_lib", - "//source/extensions/compression/gzip/compressor:compressor_lib", - "//test/test_common:utility_lib", - ], -) - -envoy_extension_cc_test( - name = "gzip_compressor_library_test", - srcs = ["gzip_compressor_library_test.cc"], - extension_name = "envoy.compression.gzip.compressor", - deps = [ "//source/extensions/compression/gzip/compressor:config", "//test/test_common:utility_lib", ], diff --git a/test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc b/test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc deleted file mode 100644 index 78395614a218..000000000000 --- a/test/extensions/compression/gzip/compressor/gzip_compressor_library_test.cc +++ /dev/null @@ -1,107 +0,0 @@ -#include "extensions/compression/gzip/compressor/config.h" - -#include "test/test_common/utility.h" - -#include "gtest/gtest.h" - -namespace Envoy { -namespace Extensions { -namespace Compression { -namespace Gzip { -namespace Compressor { - -class GzipTest : public testing::Test { -protected: - void SetUp() override { setUpGzip("{}"); } - - // GzipTest Helpers - void setUpGzip(std::string&& json) { - envoy::extensions::compression::gzip::compressor::v3::Gzip gzip; - TestUtility::loadFromJson(json, gzip); - factory_ = std::make_unique(gzip); - } - - ZlibCompressorImpl::CompressionLevel compressionLevel() const { - return factory_->compression_level_; - } - - ZlibCompressorImpl::CompressionStrategy compressionStrategy() const { - return factory_->compression_strategy_; - } - - int32_t memoryLevel() const { return factory_->memory_level_; } - int32_t windowBits() const { return factory_->window_bits_; } - int32_t chunkSize() const { return factory_->chunk_size_; } - - void expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy strategy, - absl::string_view strategy_name, - ZlibCompressorImpl::CompressionLevel level, - absl::string_view level_name) { - setUpGzip(fmt::format( - R"EOF({{"compression_strategy": "{}", "compression_level": "{}", "memory_level": 6, "window_bits": 27, "chunk_size": 10000}})EOF", - strategy_name, level_name)); - EXPECT_EQ(strategy, compressionStrategy()); - EXPECT_EQ(level, compressionLevel()); - EXPECT_EQ(6, memoryLevel()); - EXPECT_EQ(27, windowBits()); - EXPECT_EQ(10000, chunkSize()); - } - - std::unique_ptr factory_; -}; - -// Default config values. -TEST_F(GzipTest, DefaultConfigValues) { - EXPECT_EQ(5, memoryLevel()); - EXPECT_EQ(28, windowBits()); - EXPECT_EQ(4096, chunkSize()); - EXPECT_EQ(ZlibCompressorImpl::CompressionStrategy::Standard, compressionStrategy()); - EXPECT_EQ(ZlibCompressorImpl::CompressionLevel::Standard, compressionLevel()); -} - -TEST_F(GzipTest, AvailableCombinationCompressionStrategyAndLevelConfig) { - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Filtered, - "FILTERED", ZlibCompressorImpl::CompressionLevel::Best, - "BEST_COMPRESSION"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Huffman, - "HUFFMAN_ONLY", ZlibCompressorImpl::CompressionLevel::Best, - "BEST_COMPRESSION"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Rle, "RLE", - ZlibCompressorImpl::CompressionLevel::Speed, "BEST_SPEED"); - expectValidCompressionStrategyAndLevel( - ZlibCompressorImpl::CompressionStrategy::Standard, "DEFAULT_STRATEGY", - ZlibCompressorImpl::CompressionLevel::Standard, "DEFAULT_COMPRESSION"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level1, - "COMPRESSION_LEVEL_1"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level2, - "COMPRESSION_LEVEL_2"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level3, - "COMPRESSION_LEVEL_3"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level4, - "COMPRESSION_LEVEL_4"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level5, - "COMPRESSION_LEVEL_5"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level6, - "COMPRESSION_LEVEL_6"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level7, - "COMPRESSION_LEVEL_7"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level8, - "COMPRESSION_LEVEL_8"); - expectValidCompressionStrategyAndLevel(ZlibCompressorImpl::CompressionStrategy::Fixed, "FIXED", - ZlibCompressorImpl::CompressionLevel::Level9, - "COMPRESSION_LEVEL_9"); -} - -} // namespace Compressor -} // namespace Gzip -} // namespace Compression -} // namespace Extensions -} // namespace Envoy diff --git a/test/extensions/compression/gzip/compressor/zlib_compressor_impl_test.cc b/test/extensions/compression/gzip/compressor/zlib_compressor_impl_test.cc index 9d3c0fb80644..ee8b487549d6 100644 --- a/test/extensions/compression/gzip/compressor/zlib_compressor_impl_test.cc +++ b/test/extensions/compression/gzip/compressor/zlib_compressor_impl_test.cc @@ -1,6 +1,7 @@ #include "common/buffer/buffer_impl.h" #include "common/common/hex.h" +#include "extensions/compression/gzip/compressor/config.h" #include "extensions/compression/gzip/compressor/zlib_compressor_impl.h" #include "test/test_common/utility.h" @@ -15,60 +16,54 @@ namespace Gzip { namespace Compressor { namespace { -class ZlibCompressorImplTest : public testing::Test { -protected: - void expectValidFlushedBuffer(const Buffer::OwnedImpl& output_buffer) { - Buffer::RawSliceVector compressed_slices = output_buffer.getRawSlices(); - const uint64_t num_comp_slices = compressed_slices.size(); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - } +// Test helpers - void expectValidFinishedBuffer(const Buffer::OwnedImpl& output_buffer, - const uint32_t input_size) { - Buffer::RawSliceVector compressed_slices = output_buffer.getRawSlices(); - const uint64_t num_comp_slices = compressed_slices.size(); +void expectValidFlushedBuffer(const Buffer::OwnedImpl& output_buffer) { + Buffer::RawSliceVector compressed_slices = output_buffer.getRawSlices(); + const uint64_t num_comp_slices = compressed_slices.size(); - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - const std::string footer_bytes_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); - // A valid finished compressed buffer should have trailer with input size in it. - expectEqualInputSize(footer_bytes_str, input_size); - } + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); +} - void expectEqualInputSize(const std::string& footer_bytes, const uint32_t input_size) { - const std::string size_bytes = footer_bytes.substr(footer_bytes.size() - 8, 8); - uint64_t size; - StringUtil::atoull(size_bytes.c_str(), size, 16); - EXPECT_EQ(TestUtility::flipOrder(size), input_size); - } +void expectEqualInputSize(const std::string& footer_bytes, const uint32_t input_size) { + const std::string size_bytes = footer_bytes.substr(footer_bytes.size() - 8, 8); + uint64_t size; + StringUtil::atoull(size_bytes.c_str(), size, 16); + EXPECT_EQ(TestUtility::flipOrder(size), input_size); +} - void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } +void expectValidFinishedBuffer(const Buffer::OwnedImpl& output_buffer, const uint32_t input_size) { + Buffer::RawSliceVector compressed_slices = output_buffer.getRawSlices(); + const uint64_t num_comp_slices = compressed_slices.size(); - static constexpr int64_t gzip_window_bits{31}; - static constexpr int64_t memory_level{8}; - static constexpr uint64_t default_input_size{796}; -}; + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_bytes_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + + // A valid finished compressed buffer should have trailer with input size in it. + expectEqualInputSize(footer_bytes_str, input_size); +} + +void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } class ZlibCompressorImplTester : public ZlibCompressorImpl { public: @@ -82,6 +77,15 @@ class ZlibCompressorImplTester : public ZlibCompressorImpl { } }; +// Fixtures + +class ZlibCompressorImplTest : public testing::Test { +protected: + static constexpr int64_t gzip_window_bits{31}; + static constexpr int64_t memory_level{8}; + static constexpr uint64_t default_input_size{796}; +}; + class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { protected: static void compressorBadInitTestHelper(int64_t window_bits, int64_t mem_level) { @@ -110,6 +114,52 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { } }; +class ZlibCompressorImplFactoryTest + : public ::testing::TestWithParam> {}; + +INSTANTIATE_TEST_SUITE_P( + CreateCompressorTests, ZlibCompressorImplFactoryTest, + ::testing::Values(std::make_tuple("", ""), std::make_tuple("FILTERED", "BEST_COMPRESSION"), + std::make_tuple("HUFFMAN_ONLY", "BEST_COMPRESSION"), + std::make_tuple("RLE", "BEST_SPEED"), + std::make_tuple("DEFAULT_STRATEGY", "DEFAULT_COMPRESSION"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_1"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_2"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_3"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_4"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_5"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_6"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_7"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_8"), + std::make_tuple("FIXED", "COMPRESSION_LEVEL_9"))); + +TEST_P(ZlibCompressorImplFactoryTest, CreateCompressorTest) { + Buffer::OwnedImpl buffer; + envoy::extensions::compression::gzip::compressor::v3::Gzip gzip; + std::string json{"{}"}; + absl::string_view strategy = std::get<0>(GetParam()); + absl::string_view compression_level = std::get<1>(GetParam()); + + if (!strategy.empty()) { + json = fmt::format(R"EOF({{ + "compression_strategy": "{}", + "compression_level": "{}", + "memory_level": 6, + "window_bits": 27, + "chunk_size": 10000 + }})EOF", + strategy, compression_level); + } + TestUtility::loadFromJson(json, gzip); + Envoy::Compression::Compressor::CompressorPtr compressor = + GzipCompressorFactory(gzip).createCompressor(); + // Check the created compressor produces valid output. + TestUtility::feedBufferWithRandomCharacters(buffer, 4096); + compressor->compress(buffer, Envoy::Compression::Compressor::State::Flush); + expectValidFlushedBuffer(buffer); + drainBuffer(buffer); +} + // Exercises death by passing bad initialization params or by calling // compress before init. TEST_F(ZlibCompressorImplDeathTest, CompressorDeathTest) { From 63a6e46b1e8c5975b22889b254a8c88898d429c3 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 22 Apr 2020 16:37:37 +0300 Subject: [PATCH 35/56] compressor: add and use test mock for compressor Signed-off-by: Dmitry Rozhkov --- .../filters/http/common/compressor/BUILD | 1 + .../compressor/compressor_filter_test.cc | 29 ++++++--------- test/mocks/compression/compressor/BUILD | 19 ++++++++++ test/mocks/compression/compressor/mocks.cc | 22 +++++++++++ test/mocks/compression/compressor/mocks.h | 37 +++++++++++++++++++ 5 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 test/mocks/compression/compressor/BUILD create mode 100644 test/mocks/compression/compressor/mocks.cc create mode 100644 test/mocks/compression/compressor/mocks.h diff --git a/test/extensions/filters/http/common/compressor/BUILD b/test/extensions/filters/http/common/compressor/BUILD index 4d79ef96c075..af0baee55a32 100644 --- a/test/extensions/filters/http/common/compressor/BUILD +++ b/test/extensions/filters/http/common/compressor/BUILD @@ -15,6 +15,7 @@ envoy_cc_test( "//source/common/protobuf:utility_lib", "//source/extensions/compression/gzip/compressor:config", "//source/extensions/filters/http/common/compressor:compressor_lib", + "//test/mocks/compression/compressor:compressor_mocks", "//test/mocks/http:http_mocks", "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index dc375912e801..daf52035a84c 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -6,6 +6,7 @@ #include "extensions/filters/http/common/compressor/compressor.h" +#include "test/mocks/compression/compressor/mocks.h" #include "test/mocks/http/mocks.h" #include "test/mocks/protobuf/mocks.h" #include "test/mocks/runtime/mocks.h" @@ -14,17 +15,15 @@ #include "gtest/gtest.h" -using testing::Return; - namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Common { namespace Compressors { -class MockCompressor : public Envoy::Compression::Compressor::Compressor { - void compress(Buffer::Instance&, Envoy::Compression::Compressor::State) override {} -}; +using testing::_; +using testing::AtLeast; +using testing::Return; class MockCompressorFilterConfig : public CompressorFilterConfig { public: @@ -36,7 +35,9 @@ class MockCompressorFilterConfig : public CompressorFilterConfig { compressor_name) {} Envoy::Compression::Compressor::CompressorPtr makeCompressor() override { - return std::make_unique(); + auto compressor = std::make_unique(); + EXPECT_CALL(*compressor, compress(_, _)).Times(AtLeast(1)); + return compressor; } }; @@ -840,15 +841,12 @@ TEST_F(CompressorFilterTest, EtagNoCompression) { EXPECT_EQ(1, stats_.counter("test.test.not_compressed_etag").value()); } -// Verifies that compression is skipped when etag header is NOT allowed. +// Verifies that compression is not skipped when strong etag header is present. TEST_F(CompressorFilterTest, EtagCompression) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{ {":method", "get"}, {"content-length", "256"}, {"etag", "686897696a7c876b7e"}}; - feedBuffer(256); - NiceMock decoder_callbacks; - filter_->setDecoderFilterCallbacks(decoder_callbacks); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); + doResponseCompression(std::move(headers), false); EXPECT_FALSE(headers.has("etag")); EXPECT_EQ("test", headers.get_("content-encoding")); } @@ -963,8 +961,7 @@ TEST_F(CompressorFilterTest, NoVaryHeader) { filter_->setDecoderFilterCallbacks(decoder_callbacks); doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{{":method", "get"}, {"content-length", "256"}}; - feedBuffer(256); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); + doResponseCompression(std::move(headers), false); EXPECT_TRUE(headers.has("vary")); EXPECT_EQ("Accept-Encoding", headers.get_("vary")); } @@ -976,8 +973,7 @@ TEST_F(CompressorFilterTest, VaryOtherValues) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{ {":method", "get"}, {"content-length", "256"}, {"vary", "User-Agent, Cookie"}}; - feedBuffer(256); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); + doResponseCompression(std::move(headers), false); EXPECT_TRUE(headers.has("vary")); EXPECT_EQ("User-Agent, Cookie, Accept-Encoding", headers.get_("vary")); } @@ -989,8 +985,7 @@ TEST_F(CompressorFilterTest, VaryAlreadyHasAcceptEncoding) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{ {":method", "get"}, {"content-length", "256"}, {"vary", "accept-encoding"}}; - feedBuffer(256); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); + doResponseCompression(std::move(headers), false); EXPECT_TRUE(headers.has("vary")); EXPECT_EQ("accept-encoding, Accept-Encoding", headers.get_("vary")); } diff --git a/test/mocks/compression/compressor/BUILD b/test/mocks/compression/compressor/BUILD new file mode 100644 index 000000000000..e598f5cc5cf5 --- /dev/null +++ b/test/mocks/compression/compressor/BUILD @@ -0,0 +1,19 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_mock", + "envoy_package", +) + +envoy_package() + +envoy_cc_mock( + name = "compressor_mocks", + srcs = ["mocks.cc"], + hdrs = ["mocks.h"], + deps = [ + "//include/envoy/compression/compressor:compressor_config_interface", + "//include/envoy/compression/compressor:compressor_interface", + ], +) diff --git a/test/mocks/compression/compressor/mocks.cc b/test/mocks/compression/compressor/mocks.cc new file mode 100644 index 000000000000..4623d59f43f1 --- /dev/null +++ b/test/mocks/compression/compressor/mocks.cc @@ -0,0 +1,22 @@ +#include "test/mocks/compression/compressor/mocks.h" + +using testing::Invoke; +using testing::ReturnRef; + +namespace Envoy { +namespace Compression { +namespace Compressor { + +MockCompressor::MockCompressor() = default; +MockCompressor::~MockCompressor() = default; + +MockCompressorFactory::MockCompressorFactory() { + ON_CALL(*this, statsPrefix()).WillByDefault(ReturnRef(stats_prefix_)); + ON_CALL(*this, contentEncoding()).WillByDefault(ReturnRef(content_encoding_)); +} + +MockCompressorFactory::~MockCompressorFactory() = default; + +} // namespace Compressor +} // namespace Compression +} // namespace Envoy diff --git a/test/mocks/compression/compressor/mocks.h b/test/mocks/compression/compressor/mocks.h new file mode 100644 index 000000000000..e5438699484f --- /dev/null +++ b/test/mocks/compression/compressor/mocks.h @@ -0,0 +1,37 @@ +#pragma once + +#include "envoy/compression/compressor/compressor.h" +#include "envoy/compression/compressor/config.h" + +#include "gmock/gmock.h" + +namespace Envoy { +namespace Compression { +namespace Compressor { + +class MockCompressor : public Compressor { +public: + MockCompressor(); + ~MockCompressor() override; + + // Compressor::Compressor + MOCK_METHOD(void, compress, (Buffer::Instance & buffer, State state)); +}; + +class MockCompressorFactory : public CompressorFactory { +public: + MockCompressorFactory(); + ~MockCompressorFactory() override; + + // Compressor::CompressorFactory + MOCK_METHOD(CompressorPtr, createCompressor, ()); + MOCK_METHOD(const std::string&, statsPrefix, (), (const)); + MOCK_METHOD(const std::string&, contentEncoding, (), (const)); + + const std::string stats_prefix_{"mock"}; + const std::string content_encoding_{"mock"}; +}; + +} // namespace Compressor +} // namespace Compression +} // namespace Envoy From cf473a7f8c41fecd89f7bc85f4a53976d5a852f9 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 22 Apr 2020 16:57:41 +0300 Subject: [PATCH 36/56] rename MockCompressorFilterConfig to TestCompressorFilterConfig Signed-off-by: Dmitry Rozhkov --- .../compressor/compressor_filter_test.cc | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index daf52035a84c..0ffad4b40ff8 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -25,9 +25,9 @@ using testing::_; using testing::AtLeast; using testing::Return; -class MockCompressorFilterConfig : public CompressorFilterConfig { +class TestCompressorFilterConfig : public CompressorFilterConfig { public: - MockCompressorFilterConfig( + TestCompressorFilterConfig( const envoy::extensions::filters::http::compressor::v3::Compressor& compressor, const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, const std::string& compressor_name) @@ -104,7 +104,7 @@ class CompressorFilterTest : public testing::Test { envoy::extensions::filters::http::compressor::v3::Compressor compressor; TestUtility::loadFromJson(json, compressor); config_ = - std::make_shared(compressor, "test.", stats_, runtime_, "test"); + std::make_shared(compressor, "test.", stats_, runtime_, "test"); filter_ = std::make_unique(config_); filter_->setEncoderFilterCallbacks(encoder_callbacks_); } @@ -379,7 +379,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { compressor); CompressorFilterConfigSharedPtr config2; config2 = - std::make_shared(compressor, "test2.", stats, runtime, "test2"); + std::make_shared(compressor, "test2.", stats, runtime, "test2"); std::unique_ptr filter2 = std::make_unique(config2); NiceMock decoder_callbacks; filter2->setDecoderFilterCallbacks(decoder_callbacks); @@ -412,7 +412,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { compressor); CompressorFilterConfigSharedPtr config2; config2 = - std::make_shared(compressor, "test2.", stats, runtime, "gzip"); + std::make_shared(compressor, "test2.", stats, runtime, "gzip"); std::unique_ptr gzip_filter = std::make_unique(config2); NiceMock decoder_callbacks; gzip_filter->setDecoderFilterCallbacks(decoder_callbacks); @@ -441,7 +441,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { compressor); CompressorFilterConfigSharedPtr config2; config2 = - std::make_shared(compressor, "test2.", stats, runtime, "test"); + std::make_shared(compressor, "test2.", stats, runtime, "test"); std::unique_ptr filter2 = std::make_unique(config2); NiceMock decoder_callbacks; filter2->setDecoderFilterCallbacks(decoder_callbacks); @@ -470,7 +470,7 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { compressor); CompressorFilterConfigSharedPtr config2; config2 = - std::make_shared(compressor, "test2.", stats, runtime, "test"); + std::make_shared(compressor, "test2.", stats, runtime, "test"); std::unique_ptr filter2 = std::make_unique(config2); NiceMock decoder_callbacks; filter2->setDecoderFilterCallbacks(decoder_callbacks); @@ -499,11 +499,11 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { compressor); CompressorFilterConfigSharedPtr config1; config1 = - std::make_shared(compressor, "test1.", stats, runtime, "test1"); + std::make_shared(compressor, "test1.", stats, runtime, "test1"); std::unique_ptr filter1 = std::make_unique(config1); CompressorFilterConfigSharedPtr config2; config2 = - std::make_shared(compressor, "test2.", stats, runtime, "test2"); + std::make_shared(compressor, "test2.", stats, runtime, "test2"); std::unique_ptr filter2 = std::make_unique(config2); NiceMock decoder_callbacks; filter1->setDecoderFilterCallbacks(decoder_callbacks); @@ -538,11 +538,11 @@ TEST_F(CompressorFilterTest, IsAcceptEncodingAllowed) { compressor); CompressorFilterConfigSharedPtr config1; config1 = - std::make_shared(compressor, "test1.", stats, runtime, "test1"); + std::make_shared(compressor, "test1.", stats, runtime, "test1"); std::unique_ptr filter1 = std::make_unique(config1); CompressorFilterConfigSharedPtr config2; config2 = - std::make_shared(compressor, "test2.", stats, runtime, "test2"); + std::make_shared(compressor, "test2.", stats, runtime, "test2"); std::unique_ptr filter2 = std::make_unique(config2); NiceMock decoder_callbacks; filter1->setDecoderFilterCallbacks(decoder_callbacks); From 02024a33fc9ce1e134498fa185f61bb8425e501e Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 22 Apr 2020 17:33:59 +0300 Subject: [PATCH 37/56] compressor: drop gzip specific test from generic compressor Signed-off-by: Dmitry Rozhkov --- test/extensions/filters/http/compressor/BUILD | 12 -- .../compressor/gzip_compressor_filter_test.cc | 172 ------------------ 2 files changed, 184 deletions(-) delete mode 100644 test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD index 6a9f3808f52b..d192a617454f 100644 --- a/test/extensions/filters/http/compressor/BUILD +++ b/test/extensions/filters/http/compressor/BUILD @@ -11,18 +11,6 @@ load( envoy_package() -envoy_extension_cc_test( - name = "gzip_compressor_filter_test", - srcs = ["gzip_compressor_filter_test.cc"], - extension_name = "envoy.filters.http.compressor", - deps = [ - "//source/common/decompressor:decompressor_lib", - "//source/extensions/compression/gzip/compressor:config", - "//source/extensions/filters/http/compressor:compressor_filter_lib", - "//test/mocks/http:http_mocks", - ], -) - envoy_extension_cc_test( name = "compressor_filter_integration_test", srcs = [ diff --git a/test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc b/test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc deleted file mode 100644 index 423770111aca..000000000000 --- a/test/extensions/filters/http/compressor/gzip_compressor_filter_test.cc +++ /dev/null @@ -1,172 +0,0 @@ -#include - -#include "common/common/hex.h" -#include "common/decompressor/zlib_decompressor_impl.h" - -#include "extensions/compression/gzip/compressor/config.h" -#include "extensions/filters/http/compressor/compressor_filter.h" - -#include "test/mocks/http/mocks.h" -#include "test/mocks/stats/mocks.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace Compressor { - -class GzipCompressorFilterTest : public testing::Test { -protected: - void SetUp() override { - envoy::extensions::filters::http::compressor::v3::Compressor compressor; - envoy::extensions::compression::gzip::compressor::v3::Gzip gzip; - Envoy::Compression::Compressor::CompressorFactoryPtr gzip_factory = - std::make_unique(gzip); - config_ = std::make_shared(compressor, "test.", stats_, runtime_, - std::move(gzip_factory)); - filter_ = std::make_unique(config_); - filter_->setEncoderFilterCallbacks(encoder_callbacks_); - filter_->setDecoderFilterCallbacks(decoder_callbacks_); - decompressor_.init(31); - } - - // GzipCompressorFilterTest Helpers - void verifyCompressedData(const uint32_t content_length) { - // This makes sure we have a finished buffer before sending it to the client. - expectValidFinishedBuffer(content_length); - decompressor_.decompress(data_, decompressed_data_); - const std::string uncompressed_str{decompressed_data_.toString()}; - ASSERT_EQ(expected_str_.length(), uncompressed_str.length()); - EXPECT_EQ(expected_str_, uncompressed_str); - EXPECT_EQ(expected_str_.length(), - stats_.counter("test.gzip.compressor.total_uncompressed_bytes").value()); - EXPECT_EQ(data_.length(), - stats_.counter("test.gzip.compressor.total_compressed_bytes").value()); - } - - void feedBuffer(uint64_t size) { - TestUtility::feedBufferWithRandomCharacters(data_, size); - expected_str_ += data_.toString(); - } - - void drainBuffer() { - const uint64_t data_len = data_.length(); - data_.drain(data_len); - } - - void doRequest(Http::TestRequestHeaderMapImpl&& headers, bool end_stream) { - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, end_stream)); - } - - void doResponseCompression(Http::TestResponseHeaderMapImpl&& headers, bool with_trailers) { - uint64_t content_length; - ASSERT_TRUE(absl::SimpleAtoi(headers.get_("content-length"), &content_length)); - feedBuffer(content_length); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); - EXPECT_EQ("", headers.get_("content-length")); - EXPECT_EQ(Http::Headers::get().ContentEncodingValues.Gzip, headers.get_("content-encoding")); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, !with_trailers)); - if (with_trailers) { - Buffer::OwnedImpl trailers_buffer; - EXPECT_CALL(encoder_callbacks_, addEncodedData(_, true)) - .WillOnce(Invoke([&](Buffer::Instance& data, bool) { data_.move(data); })); - Http::TestResponseTrailerMapImpl trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(trailers)); - } - verifyCompressedData(content_length); - drainBuffer(); - EXPECT_EQ(1U, stats_.counter("test.gzip.compressor.compressed").value()); - } - - void expectValidFinishedBuffer(const uint32_t content_length) { - Buffer::RawSliceVector compressed_slices = data_.getRawSlices(); - const uint64_t num_comp_slices = compressed_slices.size(); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_bytes_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - - // A valid finished compressed buffer should have trailer with input size in it, i.e. equals to - // the value of content_length. - expectEqualInputSize(footer_bytes_str, content_length); - } - - void expectEqualInputSize(const std::string& footer_bytes, const uint32_t input_size) { - const std::string size_bytes = footer_bytes.substr(footer_bytes.size() - 8, 8); - uint64_t size; - StringUtil::atoull(size_bytes.c_str(), size, 16); - EXPECT_EQ(TestUtility::flipOrder(size), input_size); - } - - void doResponseNoCompression(Http::TestResponseHeaderMapImpl&& headers) { - uint64_t content_length; - ASSERT_TRUE(absl::SimpleAtoi(headers.get_("content-length"), &content_length)); - feedBuffer(content_length); - Http::TestResponseHeaderMapImpl continue_headers; - EXPECT_EQ(Http::FilterHeadersStatus::Continue, - filter_->encode100ContinueHeaders(continue_headers)); - Http::MetadataMap metadata_map{{"metadata", "metadata"}}; - EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); - EXPECT_EQ("", headers.get_("content-encoding")); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, false)); - Http::TestResponseTrailerMapImpl trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(trailers)); - EXPECT_EQ(1, stats_.counter("test.gzip.compressor.not_compressed").value()); - } - - std::shared_ptr config_; - std::unique_ptr filter_; - Buffer::OwnedImpl data_; - Decompressor::ZlibDecompressorImpl decompressor_; - Buffer::OwnedImpl decompressed_data_; - std::string expected_str_; - Stats::TestUtil::TestStore stats_; - NiceMock runtime_; - NiceMock encoder_callbacks_; - NiceMock decoder_callbacks_; -}; - -// Acceptance Testing with default configuration. -TEST_F(GzipCompressorFilterTest, AcceptanceGzipEncoding) { - doRequest({{":method", "get"}, {"accept-encoding", "deflate, gzip"}}, false); - Http::MetadataMap metadata_map{{"metadata", "metadata"}}; - EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); - Buffer::OwnedImpl data("hello"); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); - Http::TestRequestTrailerMapImpl trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); - doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); -} - -TEST_F(GzipCompressorFilterTest, AcceptanceGzipEncodingWithTrailers) { - doRequest({{":method", "get"}, {"accept-encoding", "deflate, gzip"}}, false); - Buffer::OwnedImpl data("hello"); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); - Http::TestRequestTrailerMapImpl trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); - doResponseCompression({{":method", "get"}, {"content-length", "256"}}, true); -} - -// Verifies that compression is skipped when accept-encoding header is not allowed. -TEST_F(GzipCompressorFilterTest, AcceptEncodingNoCompression) { - doRequest({{":method", "get"}, {"accept-encoding", "gzip;q=0, deflate"}}, true); - doResponseNoCompression({{":method", "get"}, {"content-length", "256"}}); -} - -// Verifies that compression is NOT skipped when accept-encoding header is allowed. -TEST_F(GzipCompressorFilterTest, AcceptEncodingCompression) { - doRequest({{":method", "get"}, {"accept-encoding", "gzip, deflate"}}, true); - doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); -} - -} // namespace Compressor -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy From 590167c41bf612ce74027cda4e023a0fa6b26c27 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 22 Apr 2020 18:03:50 +0300 Subject: [PATCH 38/56] gzip: actually remove friend class Signed-off-by: Dmitry Rozhkov --- source/extensions/compression/gzip/compressor/config.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/extensions/compression/gzip/compressor/config.h b/source/extensions/compression/gzip/compressor/config.h index ee18aca93c7a..1f062513c0fc 100644 --- a/source/extensions/compression/gzip/compressor/config.h +++ b/source/extensions/compression/gzip/compressor/config.h @@ -37,8 +37,6 @@ class GzipCompressorFactory : public Envoy::Compression::Compressor::CompressorF } private: - friend class GzipTest; - static ZlibCompressorImpl::CompressionLevel compressionLevelEnum(envoy::extensions::compression::gzip::compressor::v3::Gzip::CompressionLevel compression_level); From 680ae29b16eb477977a1ae8b988da518fdbfa6a6 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 23 Apr 2020 10:21:27 +0300 Subject: [PATCH 39/56] resort to lvalue parameter to avoid usage after std::move casting Signed-off-by: Dmitry Rozhkov --- .../compressor/compressor_filter_test.cc | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index 0ffad4b40ff8..f7877c09d37b 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -128,7 +128,7 @@ class CompressorFilterTest : public testing::Test { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, end_stream)); } - void doResponseCompression(Http::TestResponseHeaderMapImpl&& headers, bool with_trailers) { + void doResponseCompression(Http::TestResponseHeaderMapImpl& headers, bool with_trailers) { NiceMock decoder_callbacks; filter_->setDecoderFilterCallbacks(decoder_callbacks); uint64_t content_length; @@ -215,7 +215,9 @@ TEST_F(CompressorFilterTest, AcceptanceTestEncoding) { EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); Http::TestRequestTrailerMapImpl trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); - doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); + + Http::TestResponseHeaderMapImpl headers{{":method", "get"}, {"content-length", "256"}}; + doResponseCompression(headers, false); } TEST_F(CompressorFilterTest, AcceptanceTestEncodingWithTrailers) { @@ -224,7 +226,8 @@ TEST_F(CompressorFilterTest, AcceptanceTestEncodingWithTrailers) { EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); Http::TestRequestTrailerMapImpl trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); - doResponseCompression({{":method", "get"}, {"content-length", "256"}}, true); + Http::TestResponseHeaderMapImpl headers{{":method", "get"}, {"content-length", "256"}}; + doResponseCompression(headers, true); } // Verifies hasCacheControlNoTransform function. @@ -254,8 +257,9 @@ TEST_F(CompressorFilterTest, HasCacheControlNoTransformNoCompression) { // value. TEST_F(CompressorFilterTest, HasCacheControlNoTransformCompression) { doRequest({{":method", "get"}, {"accept-encoding", "test, deflate"}}, true); - doResponseCompression( - {{":method", "get"}, {"content-length", "256"}, {"cache-control", "no-cache"}}, false); + Http::TestResponseHeaderMapImpl headers{ + {":method", "get"}, {"content-length", "256"}, {"cache-control", "no-cache"}}; + doResponseCompression(headers, false); } TEST_F(CompressorFilterTest, NoAcceptEncodingHeader) { @@ -565,7 +569,8 @@ TEST_F(CompressorFilterTest, AcceptEncodingNoCompression) { // Verifies that compression is NOT skipped when accept-encoding header is allowed. TEST_F(CompressorFilterTest, AcceptEncodingCompression) { doRequest({{":method", "get"}, {"accept-encoding", "test, deflate"}}, true); - doResponseCompression({{":method", "get"}, {"content-length", "256"}}, false); + Http::TestResponseHeaderMapImpl headers{{":method", "get"}, {"content-length", "256"}}; + doResponseCompression(headers, false); } // Verifies isMinimumContentLength function. @@ -630,7 +635,8 @@ TEST_F(CompressorFilterTest, ContentLengthCompression) { } )EOF"); doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); - doResponseCompression({{":method", "get"}, {"content-length", "1000"}}, false); + Http::TestResponseHeaderMapImpl headers{{":method", "get"}, {"content-length", "1000"}}; + doResponseCompression(headers, false); } // Verifies isContentTypeAllowed function. @@ -751,10 +757,10 @@ TEST_F(CompressorFilterTest, ContentTypeNoCompression) { // Verifies that compression is NOT skipped when content-encoding header is allowed. TEST_F(CompressorFilterTest, ContentTypeCompression) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); - doResponseCompression({{":method", "get"}, - {"content-length", "256"}, - {"content-type", "application/json;charset=utf-8"}}, - false); + Http::TestResponseHeaderMapImpl headers{{":method", "get"}, + {"content-length", "256"}, + {"content-type", "application/json;charset=utf-8"}}; + doResponseCompression(headers, false); } // Verifies sanitizeEtagHeader function. @@ -846,7 +852,7 @@ TEST_F(CompressorFilterTest, EtagCompression) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{ {":method", "get"}, {"content-length", "256"}, {"etag", "686897696a7c876b7e"}}; - doResponseCompression(std::move(headers), false); + doResponseCompression(headers, false); EXPECT_FALSE(headers.has("etag")); EXPECT_EQ("test", headers.get_("content-encoding")); } @@ -890,8 +896,9 @@ TEST_F(CompressorFilterTest, IsTransferEncodingAllowed) { // Tests compression when Transfer-Encoding header exists. TEST_F(CompressorFilterTest, TransferEncodingChunked) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); - doResponseCompression( - {{":method", "get"}, {"content-length", "256"}, {"transfer-encoding", "chunked"}}, false); + Http::TestResponseHeaderMapImpl headers{ + {":method", "get"}, {"content-length", "256"}, {"transfer-encoding", "chunked"}}; + doResponseCompression(headers, false); } // Tests compression when Transfer-Encoding header exists. @@ -961,7 +968,7 @@ TEST_F(CompressorFilterTest, NoVaryHeader) { filter_->setDecoderFilterCallbacks(decoder_callbacks); doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{{":method", "get"}, {"content-length", "256"}}; - doResponseCompression(std::move(headers), false); + doResponseCompression(headers, false); EXPECT_TRUE(headers.has("vary")); EXPECT_EQ("Accept-Encoding", headers.get_("vary")); } @@ -973,7 +980,7 @@ TEST_F(CompressorFilterTest, VaryOtherValues) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{ {":method", "get"}, {"content-length", "256"}, {"vary", "User-Agent, Cookie"}}; - doResponseCompression(std::move(headers), false); + doResponseCompression(headers, false); EXPECT_TRUE(headers.has("vary")); EXPECT_EQ("User-Agent, Cookie, Accept-Encoding", headers.get_("vary")); } @@ -985,7 +992,7 @@ TEST_F(CompressorFilterTest, VaryAlreadyHasAcceptEncoding) { doRequest({{":method", "get"}, {"accept-encoding", "test"}}, true); Http::TestResponseHeaderMapImpl headers{ {":method", "get"}, {"content-length", "256"}, {"vary", "accept-encoding"}}; - doResponseCompression(std::move(headers), false); + doResponseCompression(headers, false); EXPECT_TRUE(headers.has("vary")); EXPECT_EQ("accept-encoding, Accept-Encoding", headers.get_("vary")); } From dd93ad69a2100373a9b127506afb1533746a7df5 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 23 Apr 2020 10:55:06 +0300 Subject: [PATCH 40/56] test: set expectations of compress() call counts for every test case Signed-off-by: Dmitry Rozhkov --- .../http/common/compressor/compressor_filter_test.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc index f7877c09d37b..225edcaeaa32 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_test.cc @@ -22,7 +22,6 @@ namespace Common { namespace Compressors { using testing::_; -using testing::AtLeast; using testing::Return; class TestCompressorFilterConfig : public CompressorFilterConfig { @@ -36,9 +35,14 @@ class TestCompressorFilterConfig : public CompressorFilterConfig { Envoy::Compression::Compressor::CompressorPtr makeCompressor() override { auto compressor = std::make_unique(); - EXPECT_CALL(*compressor, compress(_, _)).Times(AtLeast(1)); + EXPECT_CALL(*compressor, compress(_, _)).Times(expected_compress_calls_); return compressor; } + + void setExpectedCompressCalls(uint32_t calls) { expected_compress_calls_ = calls; } + +private: + uint32_t expected_compress_calls_{1}; }; class CompressorFilterTest : public testing::Test { @@ -169,7 +173,7 @@ class CompressorFilterTest : public testing::Test { EXPECT_EQ(1, stats_.counter("test.test.not_compressed").value()); } - CompressorFilterConfigSharedPtr config_; + std::shared_ptr config_; std::unique_ptr filter_; Buffer::OwnedImpl data_; std::string expected_str_; @@ -227,6 +231,7 @@ TEST_F(CompressorFilterTest, AcceptanceTestEncodingWithTrailers) { Http::TestRequestTrailerMapImpl trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); Http::TestResponseHeaderMapImpl headers{{":method", "get"}, {"content-length", "256"}}; + config_->setExpectedCompressCalls(2); doResponseCompression(headers, true); } From e6f9c792ad65fd7f906650bbb0b296e408e858f1 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 23 Apr 2020 12:23:36 +0300 Subject: [PATCH 41/56] compressor: remove unused declaration from mock Signed-off-by: Dmitry Rozhkov --- test/mocks/compression/compressor/mocks.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/mocks/compression/compressor/mocks.cc b/test/mocks/compression/compressor/mocks.cc index 4623d59f43f1..9d2ff9cac79e 100644 --- a/test/mocks/compression/compressor/mocks.cc +++ b/test/mocks/compression/compressor/mocks.cc @@ -1,6 +1,5 @@ #include "test/mocks/compression/compressor/mocks.h" -using testing::Invoke; using testing::ReturnRef; namespace Envoy { From 9aaf9ff5692a59500db04f24e3f6cb245ff52a32 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 23 Apr 2020 12:27:56 +0300 Subject: [PATCH 42/56] compressor: add test case for CompressorFilterConfig Signed-off-by: Dmitry Rozhkov --- test/extensions/filters/http/compressor/BUILD | 14 ++++++++ .../http/compressor/compressor_filter_test.cc | 34 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 test/extensions/filters/http/compressor/compressor_filter_test.cc diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD index d192a617454f..87a1a23d04f2 100644 --- a/test/extensions/filters/http/compressor/BUILD +++ b/test/extensions/filters/http/compressor/BUILD @@ -11,6 +11,20 @@ load( envoy_package() +envoy_extension_cc_test( + name = "compressor_filter_test", + srcs = [ + "compressor_filter_test.cc", + ], + extension_name = "envoy.filters.http.compressor", + deps = [ + "//source/extensions/filters/http/compressor:compressor_filter_lib", + "//test/mocks/compression/compressor:compressor_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:utility_lib", + ], +) + envoy_extension_cc_test( name = "compressor_filter_integration_test", srcs = [ diff --git a/test/extensions/filters/http/compressor/compressor_filter_test.cc b/test/extensions/filters/http/compressor/compressor_filter_test.cc new file mode 100644 index 000000000000..a8f2571f6266 --- /dev/null +++ b/test/extensions/filters/http/compressor/compressor_filter_test.cc @@ -0,0 +1,34 @@ +#include "extensions/filters/http/compressor/compressor_filter.h" + +#include "test/mocks/compression/compressor/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/stats/mocks.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Compressor { +namespace { + +using testing::NiceMock; + +TEST(CompressorFilterConfigTests, MakeCompressorTest) { + const envoy::extensions::filters::http::compressor::v3::Compressor compressor_cfg; + NiceMock runtime; + Stats::TestUtil::TestStore stats; + auto compressor_factory(std::make_unique()); + EXPECT_CALL(*compressor_factory, createCompressor()).Times(1); + EXPECT_CALL(*compressor_factory, statsPrefix()).Times(1); + EXPECT_CALL(*compressor_factory, contentEncoding()).Times(1); + CompressorFilterConfig config(compressor_cfg, "test.compressor.", stats, runtime, + std::move(compressor_factory)); + Envoy::Compression::Compressor::CompressorPtr compressor = config.makeCompressor(); +} + +} // namespace +} // namespace Compressor +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy From d9ee8e048794b10c55003ff1bbc2412b570c33ef Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 23 Apr 2020 13:09:57 +0300 Subject: [PATCH 43/56] add compressor filter specific stats prefix Signed-off-by: Dmitry Rozhkov --- source/extensions/filters/http/compressor/compressor_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index bb37336fbbd0..94111db4bbb3 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -10,7 +10,7 @@ CompressorFilterConfig::CompressorFilterConfig( const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, Compression::Compressor::CompressorFactoryPtr compressor_factory) : Common::Compressors::CompressorFilterConfig( - generic_compressor, stats_prefix + compressor_factory->statsPrefix(), scope, runtime, + generic_compressor, stats_prefix + "compressor." + compressor_factory->statsPrefix(), scope, runtime, compressor_factory->contentEncoding()), compressor_factory_(std::move(compressor_factory)) {} From 503c81aa374383addc3e7577ea910ec101a8418e Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 23 Apr 2020 15:59:26 +0300 Subject: [PATCH 44/56] fix code formatting Signed-off-by: Dmitry Rozhkov --- .../extensions/filters/http/compressor/compressor_filter.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index 94111db4bbb3..16593dbfd11a 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -10,8 +10,8 @@ CompressorFilterConfig::CompressorFilterConfig( const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, Compression::Compressor::CompressorFactoryPtr compressor_factory) : Common::Compressors::CompressorFilterConfig( - generic_compressor, stats_prefix + "compressor." + compressor_factory->statsPrefix(), scope, runtime, - compressor_factory->contentEncoding()), + generic_compressor, stats_prefix + "compressor." + compressor_factory->statsPrefix(), + scope, runtime, compressor_factory->contentEncoding()), compressor_factory_(std::move(compressor_factory)) {} Envoy::Compression::Compressor::CompressorPtr CompressorFilterConfig::makeCompressor() { From 6ede080c7d82051e42627307ecf90d3c7ebb7b7c Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 27 Apr 2020 14:25:22 +0300 Subject: [PATCH 45/56] drop removed component from CODEOWNERS Signed-off-by: Dmitry Rozhkov --- CODEOWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index d52fa76d50c4..b82652d16143 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -26,7 +26,6 @@ extensions/filters/common/original_src @snowp @klarose # compressor used by http compression filters /*/extensions/filters/http/common/compressor @gsagula @rojkov @dio /*/extensions/filters/http/compressor @rojkov @dio -/*/extensions/filters/http/compressor/gzip @gsagula @rojkov @dio # jwt_authn http filter extension /*/extensions/filters/http/jwt_authn @qiwzhang @lizan # grpc_http1_reverse_bridge http filter extension From c5149d222c37a7c5b1e382e46e66f9f780f9f472 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 27 Apr 2020 14:30:58 +0300 Subject: [PATCH 46/56] docs: fix doc title Signed-off-by: Dmitry Rozhkov --- .../compression/common/compressor/v3/compressor.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/compression/common/compressor/v3/compressor.proto b/api/envoy/extensions/compression/common/compressor/v3/compressor.proto index f3259806b5f0..a0df58519c6c 100644 --- a/api/envoy/extensions/compression/common/compressor/v3/compressor.proto +++ b/api/envoy/extensions/compression/common/compressor/v3/compressor.proto @@ -13,7 +13,7 @@ option java_outer_classname = "CompressorProto"; option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; -// [#protodoc-title: CompressorLibrary] +// [#protodoc-title: Compressor Library] message CompressorLibrary { // Compressor library specific configuration. See the supported libraries for further From 9b59cac3170e0004685fc82509520cee830f9636 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 27 Apr 2020 14:31:38 +0300 Subject: [PATCH 47/56] run check_format.py fix Signed-off-by: Dmitry Rozhkov --- source/extensions/filters/http/gzip/gzip_filter.cc | 3 ++- .../http/common/compressor/compressor_filter_speed_test.cc | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index e9ad28233ccf..1d271f72752b 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -34,7 +34,8 @@ GzipFilterConfig::GzipFilterConfig(const envoy::extensions::filters::http::gzip: chunk_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(gzip, chunk_size, 4096)) {} Envoy::Compression::Compressor::CompressorPtr GzipFilterConfig::makeCompressor() { - auto compressor = std::make_unique(chunk_size_); + auto compressor = + std::make_unique(chunk_size_); compressor->init(compressionLevel(), compressionStrategy(), windowBits(), memoryLevel()); return compressor; } diff --git a/test/extensions/filters/http/common/compressor/compressor_filter_speed_test.cc b/test/extensions/filters/http/common/compressor/compressor_filter_speed_test.cc index 29ecab430220..cb3c4b26e425 100644 --- a/test/extensions/filters/http/common/compressor/compressor_filter_speed_test.cc +++ b/test/extensions/filters/http/common/compressor/compressor_filter_speed_test.cc @@ -25,8 +25,8 @@ class MockCompressorFilterConfig : public CompressorFilterConfig { const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, const std::string& compressor_name, Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel level, - Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy strategy, int64_t window_bits, - uint64_t memory_level) + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy strategy, + int64_t window_bits, uint64_t memory_level) : CompressorFilterConfig(compressor, stats_prefix + compressor_name + ".", scope, runtime, compressor_name), level_(level), strategy_(strategy), window_bits_(window_bits), memory_level_(memory_level) { @@ -46,7 +46,8 @@ class MockCompressorFilterConfig : public CompressorFilterConfig { using CompressionParams = std::tuple; + Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy, int64_t, + uint64_t>; static constexpr uint64_t TestDataSize = 122880; From a271f28a48f657dde50f5aedf93e60dc407f4c1e Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 27 Apr 2020 15:11:55 +0300 Subject: [PATCH 48/56] gzip: do protobuf conversion with JSON round trip Signed-off-by: Dmitry Rozhkov --- .../filters/http/gzip/gzip_filter.cc | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index 1d271f72752b..8d1e45824eef 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -3,6 +3,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "common/http/headers.h" +#include "common/protobuf/message_validator_impl.h" #include "common/protobuf/protobuf.h" namespace Envoy { @@ -84,29 +85,9 @@ GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip: // compressor_library field which is mandatory in v3. Here we convert gzip's Compressor to // what is accepted by the generic compressor. if (gzip.has_compressor()) { - compressor.set_allocated_content_length( - // According to - // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage - // the message Compressor takes ownership of the allocated Protobuf::Uint32Value object. - new Protobuf::UInt32Value(gzip.compressor().content_length())); - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - for (const std::string& ctype : gzip.compressor().content_type()) { - compressor.add_content_type(ctype); - } - compressor.set_disable_on_etag_header(gzip.compressor().disable_on_etag_header()); - compressor.set_remove_accept_encoding_header(gzip.compressor().remove_accept_encoding_header()); - if (gzip.compressor().has_runtime_enabled()) { - auto feature_flag(new envoy::config::core::v3::RuntimeFeatureFlag()); - feature_flag->set_runtime_key(gzip.compressor().runtime_enabled().runtime_key()); - feature_flag->set_allocated_default_value( - // According to - // https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#embeddedmessage - // the message RuntimeFeatureFlag takes ownership of the allocated Protobuf::BoolValue - // object. - new Protobuf::BoolValue(gzip.compressor().runtime_enabled().default_value())); - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - compressor.set_allocated_runtime_enabled(feature_flag); - } + ProtobufWkt::Struct tmp; + MessageUtil::jsonConvert(gzip.compressor(), tmp); + MessageUtil::jsonConvert(tmp, ProtobufMessage::getStrictValidationVisitor(), compressor); return compressor; } From bc77b2521ea17cdc1477e7984ee39cdbc1db7584 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 27 Apr 2020 15:50:45 +0300 Subject: [PATCH 49/56] run proto_format.sh fix Signed-off-by: Dmitry Rozhkov --- .../compression/common/compressor/v3/compressor.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto index f3259806b5f0..a0df58519c6c 100644 --- a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto @@ -13,7 +13,7 @@ option java_outer_classname = "CompressorProto"; option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; -// [#protodoc-title: CompressorLibrary] +// [#protodoc-title: Compressor Library] message CompressorLibrary { // Compressor library specific configuration. See the supported libraries for further From 4a9bff79cba4a11787fbfec3102c0080acc99684 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 28 Apr 2020 10:54:49 +0300 Subject: [PATCH 50/56] reduce gzip compressor's stats prefix to just gzip Signed-off-by: Dmitry Rozhkov --- source/extensions/compression/gzip/compressor/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/compression/gzip/compressor/config.h b/source/extensions/compression/gzip/compressor/config.h index 1f062513c0fc..25c96fff8a90 100644 --- a/source/extensions/compression/gzip/compressor/config.h +++ b/source/extensions/compression/gzip/compressor/config.h @@ -18,7 +18,7 @@ namespace Compressor { namespace { -const std::string& gzipStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "gzip.compressor."); } +const std::string& gzipStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "gzip."); } const std::string& gzipExtensionName() { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.compression.gzip.compressor"); } From 5f683bd66fe225e14e67c4ed6f89cb3d968d5b08 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 4 May 2020 10:51:13 +0300 Subject: [PATCH 51/56] fix reference to the interface class Signed-off-by: Dmitry Rozhkov --- .../compression/gzip/compressor/zlib_compressor_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h index d49901e434d9..4b2956688ef4 100644 --- a/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h +++ b/source/extensions/compression/gzip/compressor/zlib_compressor_impl.h @@ -82,7 +82,7 @@ class ZlibCompressorImpl : public Zlib::Base, public Envoy::Compression::Compres void init(CompressionLevel level, CompressionStrategy strategy, int64_t window_bits, uint64_t memory_level); - // Compressor + // Compression::Compressor::Compressor void compress(Buffer::Instance& buffer, Envoy::Compression::Compressor::State state) override; private: From 786ab773ccfd54e0ed224133f974ae6b3d6bf0a9 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Mon, 4 May 2020 10:57:03 +0300 Subject: [PATCH 52/56] avoid changing frozen API in gzip.proto Signed-off-by: Dmitry Rozhkov --- .../http/compressor/v3/compressor.proto | 4 +-- .../extensions/filters/http/gzip/v3/BUILD | 2 +- .../filters/http/gzip/v3/gzip.proto | 36 ++----------------- .../http/compressor/v3/compressor.proto | 4 +-- .../extensions/filters/http/gzip/v3/BUILD | 2 +- .../filters/http/gzip/v3/gzip.proto | 36 ++----------------- .../filters/http/compressor/config.cc | 5 +++ .../filters/http/gzip/gzip_filter.cc | 12 ++----- 8 files changed, 17 insertions(+), 84 deletions(-) diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 3f07faec505f..1f4ea2d65e49 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -55,6 +55,6 @@ message Compressor { // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. - compression.common.compressor.v3.CompressorLibrary compressor_library = 6 - [(validate.rules).message = {required: true}]; + // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. + compression.common.compressor.v3.CompressorLibrary compressor_library = 6; } diff --git a/api/envoy/extensions/filters/http/gzip/v3/BUILD b/api/envoy/extensions/filters/http/gzip/v3/BUILD index 6bd8f8337ac8..1c3cf3aef2b5 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/BUILD +++ b/api/envoy/extensions/filters/http/gzip/v3/BUILD @@ -6,8 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/gzip/v2:pkg", + "//envoy/extensions/filters/http/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto index 4a0b5c768447..20cae5c400d3 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/api/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.gzip.v3; -import "envoy/config/core/v3/base.proto"; +import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; import "google/protobuf/wrappers.proto"; @@ -75,42 +75,10 @@ message Gzip { // Set of configuration parameters common for all compression filters. If this field is set then // the fields `content_length`, `content_type`, `disable_on_etag_header` and // `remove_accept_encoding_header` are ignored. - Compressor compressor = 10; + compressor.v3.Compressor compressor = 10; // Value for Zlib's next output buffer. If not set, defaults to 4096. // See https://www.zlib.net/manual.html for more details. Also see // https://github.com/envoyproxy/envoy/issues/8448 for context on this filter's performance. google.protobuf.UInt32Value chunk_size = 11 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; } - -// A locally defined Compressor type to use in the gzip filter since the generic Compressor requires -// setting the compressor_library field which doesn't make sense in the context of the deprecated -// gzip filter. -// [#next-free-field: 6] -message Compressor { - // Minimum response length, in bytes, which will trigger compression. The default value is 30. - google.protobuf.UInt32Value content_length = 1; - - // Set of strings that allows specifying which mime-types yield compression; e.g., - // application/json, text/html, etc. When this field is not defined, compression will be applied - // to the following mime-types: "application/javascript", "application/json", - // "application/xhtml+xml", "image/svg+xml", "text/css", "text/html", "text/plain", "text/xml" - // and their synonyms. - repeated string content_type = 2; - - // If true, disables compression when the response contains an etag header. When it is false, the - // filter will preserve weak etags and remove the ones that require strong validation. - bool disable_on_etag_header = 3; - - // If true, removes accept-encoding from the request headers before dispatching it to the upstream - // so that responses do not get compressed before reaching the filter. - // .. attention: - // - // To avoid interfering with other compression filters in the same chain use this option in - // the filter closest to the upstream. - bool remove_accept_encoding_header = 4; - - // Runtime flag that controls whether the filter is enabled or not. If set to false, the - // filter will operate as a pass-through filter. If not specified, defaults to enabled. - config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; -} diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 3f07faec505f..1f4ea2d65e49 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -55,6 +55,6 @@ message Compressor { // A compressor library to use for compression. Currently only // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. - compression.common.compressor.v3.CompressorLibrary compressor_library = 6 - [(validate.rules).message = {required: true}]; + // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. + compression.common.compressor.v3.CompressorLibrary compressor_library = 6; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD index 6bd8f8337ac8..1c3cf3aef2b5 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/BUILD @@ -6,8 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/gzip/v2:pkg", + "//envoy/extensions/filters/http/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto index 34c3e9569e9e..e711827481a3 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/gzip/v3/gzip.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.gzip.v3; -import "envoy/config/core/v3/base.proto"; +import "envoy/extensions/filters/http/compressor/v3/compressor.proto"; import "google/protobuf/wrappers.proto"; @@ -70,7 +70,7 @@ message Gzip { // Set of configuration parameters common for all compression filters. If this field is set then // the fields `content_length`, `content_type`, `disable_on_etag_header` and // `remove_accept_encoding_header` are ignored. - Compressor compressor = 10; + compressor.v3.Compressor compressor = 10; // Value for Zlib's next output buffer. If not set, defaults to 4096. // See https://www.zlib.net/manual.html for more details. Also see @@ -85,35 +85,3 @@ message Gzip { bool hidden_envoy_deprecated_remove_accept_encoding_header = 8 [deprecated = true]; } - -// A locally defined Compressor type to use in the gzip filter since the generic Compressor requires -// setting the compressor_library field which doesn't make sense in the context of the deprecated -// gzip filter. -// [#next-free-field: 6] -message Compressor { - // Minimum response length, in bytes, which will trigger compression. The default value is 30. - google.protobuf.UInt32Value content_length = 1; - - // Set of strings that allows specifying which mime-types yield compression; e.g., - // application/json, text/html, etc. When this field is not defined, compression will be applied - // to the following mime-types: "application/javascript", "application/json", - // "application/xhtml+xml", "image/svg+xml", "text/css", "text/html", "text/plain", "text/xml" - // and their synonyms. - repeated string content_type = 2; - - // If true, disables compression when the response contains an etag header. When it is false, the - // filter will preserve weak etags and remove the ones that require strong validation. - bool disable_on_etag_header = 3; - - // If true, removes accept-encoding from the request headers before dispatching it to the upstream - // so that responses do not get compressed before reaching the filter. - // .. attention: - // - // To avoid interfering with other compression filters in the same chain use this option in - // the filter closest to the upstream. - bool remove_accept_encoding_header = 4; - - // Runtime flag that controls whether the filter is enabled or not. If set to false, the - // filter will operate as a pass-through filter. If not specified, defaults to enabled. - config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; -} diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc index 1fb2c7f55c61..aff3ca5afe4c 100644 --- a/source/extensions/filters/http/compressor/config.cc +++ b/source/extensions/filters/http/compressor/config.cc @@ -14,6 +14,11 @@ namespace Compressor { Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::compressor::v3::Compressor& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + // TODO(rojkov): instead of throwing an exception make the Compressor.compressor_library field + // required when the Gzip HTTP-filter is fully deprecated and removed. + if (!proto_config.has_compressor_library()) { + throw EnvoyException("Compressor filter doesn't have compressor_library defined"); + } const std::string type{TypeUtil::typeUrlToDescriptorFullName( proto_config.compressor_library().typed_config().type_url())}; Compression::Compressor::NamedCompressorLibraryConfigFactory* const config_factory = diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index 8d1e45824eef..b0b0ab06aec9 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -79,18 +79,10 @@ uint64_t GzipFilterConfig::windowBitsUint(Protobuf::uint32 window_bits) { const envoy::extensions::filters::http::compressor::v3::Compressor GzipFilterConfig::compressorConfig(const envoy::extensions::filters::http::gzip::v3::Gzip& gzip) { - envoy::extensions::filters::http::compressor::v3::Compressor compressor = {}; - - // The deprecated gzip filter is using a locally defined Compressor field to avoid setting the - // compressor_library field which is mandatory in v3. Here we convert gzip's Compressor to - // what is accepted by the generic compressor. if (gzip.has_compressor()) { - ProtobufWkt::Struct tmp; - MessageUtil::jsonConvert(gzip.compressor(), tmp); - MessageUtil::jsonConvert(tmp, ProtobufMessage::getStrictValidationVisitor(), compressor); - return compressor; + return gzip.compressor(); } - + envoy::extensions::filters::http::compressor::v3::Compressor compressor = {}; if (gzip.has_hidden_envoy_deprecated_content_length()) { compressor.set_allocated_content_length( // According to From 4b4cfdafffd81213babff68e296d8d0d0bebba50 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 5 May 2020 11:20:05 +0300 Subject: [PATCH 53/56] compressor: add stats_prefix to compressor filter Signed-off-by: Dmitry Rozhkov --- .../filters/http/compressor/v3/compressor.proto | 9 ++++++++- .../filters/http/compressor/v3/compressor.proto | 9 ++++++++- .../filters/http/compressor/compressor_filter.cc | 4 +++- .../compressor/compressor_filter_integration_test.cc | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 1f4ea2d65e49..4e319170048d 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -21,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Compressor :ref:`configuration overview `. // [#extension: envoy.filters.http.compressor] -// [#next-free-field: 7] +// [#next-free-field: 8] message Compressor { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.Compressor"; @@ -57,4 +57,11 @@ message Compressor { // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. compression.common.compressor.v3.CompressorLibrary compressor_library = 6; + + // Prefix added to the names of stats generated by this filter. It can be used to disambiguate stats + // trees from different compression filters. For example stats of a compressor filter limited to compress + // binaries blobs and tweaked accordingly can be prefixed with "bins.". And another compressor filter + // tweaked co compress text content only can be prefixed with "text.". + // This field is ignored if used in the context of the gzip http-filter. + string stats_prefix = 7; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 1f4ea2d65e49..4e319170048d 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -21,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Compressor :ref:`configuration overview `. // [#extension: envoy.filters.http.compressor] -// [#next-free-field: 7] +// [#next-free-field: 8] message Compressor { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.Compressor"; @@ -57,4 +57,11 @@ message Compressor { // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. compression.common.compressor.v3.CompressorLibrary compressor_library = 6; + + // Prefix added to the names of stats generated by this filter. It can be used to disambiguate stats + // trees from different compression filters. For example stats of a compressor filter limited to compress + // binaries blobs and tweaked accordingly can be prefixed with "bins.". And another compressor filter + // tweaked co compress text content only can be prefixed with "text.". + // This field is ignored if used in the context of the gzip http-filter. + string stats_prefix = 7; } diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index 16593dbfd11a..dd0d68d6d6a1 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -10,7 +10,9 @@ CompressorFilterConfig::CompressorFilterConfig( const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, Compression::Compressor::CompressorFactoryPtr compressor_factory) : Common::Compressors::CompressorFilterConfig( - generic_compressor, stats_prefix + "compressor." + compressor_factory->statsPrefix(), + generic_compressor, + stats_prefix + "compressor." + generic_compressor.stats_prefix() + + compressor_factory->statsPrefix(), scope, runtime, compressor_factory->contentEncoding()), compressor_factory_(std::move(compressor_factory)) {} diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 92c4ca5b3657..c364382b414e 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -81,6 +81,7 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Fri, 8 May 2020 12:04:10 +0300 Subject: [PATCH 54/56] Revert "compressor: add stats_prefix to compressor filter" This reverts commit 4b4cfdafffd81213babff68e296d8d0d0bebba50. Signed-off-by: Dmitry Rozhkov --- .../filters/http/compressor/v3/compressor.proto | 9 +-------- .../filters/http/compressor/v3/compressor.proto | 9 +-------- .../filters/http/compressor/compressor_filter.cc | 4 +--- .../compressor/compressor_filter_integration_test.cc | 1 - 4 files changed, 3 insertions(+), 20 deletions(-) diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 4e319170048d..1f4ea2d65e49 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -21,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Compressor :ref:`configuration overview `. // [#extension: envoy.filters.http.compressor] -// [#next-free-field: 8] +// [#next-free-field: 7] message Compressor { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.Compressor"; @@ -57,11 +57,4 @@ message Compressor { // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. compression.common.compressor.v3.CompressorLibrary compressor_library = 6; - - // Prefix added to the names of stats generated by this filter. It can be used to disambiguate stats - // trees from different compression filters. For example stats of a compressor filter limited to compress - // binaries blobs and tweaked accordingly can be prefixed with "bins.". And another compressor filter - // tweaked co compress text content only can be prefixed with "text.". - // This field is ignored if used in the context of the gzip http-filter. - string stats_prefix = 7; } diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 4e319170048d..1f4ea2d65e49 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -21,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Compressor :ref:`configuration overview `. // [#extension: envoy.filters.http.compressor] -// [#next-free-field: 8] +// [#next-free-field: 7] message Compressor { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.compressor.v2.Compressor"; @@ -57,11 +57,4 @@ message Compressor { // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. compression.common.compressor.v3.CompressorLibrary compressor_library = 6; - - // Prefix added to the names of stats generated by this filter. It can be used to disambiguate stats - // trees from different compression filters. For example stats of a compressor filter limited to compress - // binaries blobs and tweaked accordingly can be prefixed with "bins.". And another compressor filter - // tweaked co compress text content only can be prefixed with "text.". - // This field is ignored if used in the context of the gzip http-filter. - string stats_prefix = 7; } diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index dd0d68d6d6a1..16593dbfd11a 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -10,9 +10,7 @@ CompressorFilterConfig::CompressorFilterConfig( const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, Compression::Compressor::CompressorFactoryPtr compressor_factory) : Common::Compressors::CompressorFilterConfig( - generic_compressor, - stats_prefix + "compressor." + generic_compressor.stats_prefix() + - compressor_factory->statsPrefix(), + generic_compressor, stats_prefix + "compressor." + compressor_factory->statsPrefix(), scope, runtime, compressor_factory->contentEncoding()), compressor_factory_(std::move(compressor_factory)) {} diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index c364382b414e..92c4ca5b3657 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -81,7 +81,6 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Fri, 8 May 2020 10:26:59 +0300 Subject: [PATCH 55/56] replace CompressorLibrary with TypedExtensionConfig Signed-off-by: Dmitry Rozhkov --- api/BUILD | 1 - .../compression/common/compressor/v3/BUILD | 9 -------- .../common/compressor/v3/compressor.proto | 22 ------------------- .../filters/http/compressor/v3/BUILD | 1 - .../http/compressor/v3/compressor.proto | 4 ++-- api/versioning/BUILD | 1 - .../api-v3/config/compression/compression.rst | 1 - .../compression/common/compressor/v3/BUILD | 9 -------- .../common/compressor/v3/compressor.proto | 22 ------------------- .../filters/http/compressor/v3/BUILD | 1 - .../http/compressor/v3/compressor.proto | 4 ++-- .../http/compressor/compressor_filter.cc | 4 +++- .../compressor_filter_integration_test.cc | 2 ++ 13 files changed, 9 insertions(+), 72 deletions(-) delete mode 100644 api/envoy/extensions/compression/common/compressor/v3/BUILD delete mode 100644 api/envoy/extensions/compression/common/compressor/v3/compressor.proto delete mode 100644 generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD delete mode 100644 generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto diff --git a/api/BUILD b/api/BUILD index fac3c4edb2d1..2472352ab2bd 100644 --- a/api/BUILD +++ b/api/BUILD @@ -161,7 +161,6 @@ proto_library( "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", - "//envoy/extensions/compression/common/compressor/v3:pkg", "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", "//envoy/extensions/filters/http/adaptive_concurrency/v3:pkg", diff --git a/api/envoy/extensions/compression/common/compressor/v3/BUILD b/api/envoy/extensions/compression/common/compressor/v3/BUILD deleted file mode 100644 index ef3541ebcb1d..000000000000 --- a/api/envoy/extensions/compression/common/compressor/v3/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -# DO NOT EDIT. This file is generated by tools/proto_sync.py. - -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], -) diff --git a/api/envoy/extensions/compression/common/compressor/v3/compressor.proto b/api/envoy/extensions/compression/common/compressor/v3/compressor.proto deleted file mode 100644 index a0df58519c6c..000000000000 --- a/api/envoy/extensions/compression/common/compressor/v3/compressor.proto +++ /dev/null @@ -1,22 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.compression.common.compressor.v3; - -import "google/protobuf/any.proto"; - -import "udpa/annotations/migrate.proto"; -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.compression.common.compressor.v3"; -option java_outer_classname = "CompressorProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: Compressor Library] - -message CompressorLibrary { - // Compressor library specific configuration. See the supported libraries for further - // documentation. - google.protobuf.Any typed_config = 1; -} diff --git a/api/envoy/extensions/filters/http/compressor/v3/BUILD b/api/envoy/extensions/filters/http/compressor/v3/BUILD index 11ff14b0ad5f..8dc07faa8f9f 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/BUILD +++ b/api/envoy/extensions/filters/http/compressor/v3/BUILD @@ -8,7 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/compressor/v2:pkg", - "//envoy/extensions/compression/common/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index 1f4ea2d65e49..1f6cd63e9d52 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; -import "envoy/extensions/compression/common/compressor/v3/compressor.proto"; +import "envoy/config/core/v3/extension.proto"; import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; @@ -56,5 +56,5 @@ message Compressor { // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. - compression.common.compressor.v3.CompressorLibrary compressor_library = 6; + config.core.v3.TypedExtensionConfig compressor_library = 6; } diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 8c2650c66bc1..4d768d09a015 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -44,7 +44,6 @@ proto_library( "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", - "//envoy/extensions/compression/common/compressor/v3:pkg", "//envoy/extensions/compression/gzip/compressor/v3:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", "//envoy/extensions/filters/http/adaptive_concurrency/v3:pkg", diff --git a/docs/root/api-v3/config/compression/compression.rst b/docs/root/api-v3/config/compression/compression.rst index ae7b178bfda0..80aa0ba927cc 100644 --- a/docs/root/api-v3/config/compression/compression.rst +++ b/docs/root/api-v3/config/compression/compression.rst @@ -5,5 +5,4 @@ Compression :glob: :maxdepth: 2 - ../../extensions/compression/common/*/v3/* ../../extensions/compression/gzip/*/v3/* diff --git a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD deleted file mode 100644 index ef3541ebcb1d..000000000000 --- a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -# DO NOT EDIT. This file is generated by tools/proto_sync.py. - -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], -) diff --git a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto deleted file mode 100644 index a0df58519c6c..000000000000 --- a/generated_api_shadow/envoy/extensions/compression/common/compressor/v3/compressor.proto +++ /dev/null @@ -1,22 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.compression.common.compressor.v3; - -import "google/protobuf/any.proto"; - -import "udpa/annotations/migrate.proto"; -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.compression.common.compressor.v3"; -option java_outer_classname = "CompressorProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: Compressor Library] - -message CompressorLibrary { - // Compressor library specific configuration. See the supported libraries for further - // documentation. - google.protobuf.Any typed_config = 1; -} diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD index 11ff14b0ad5f..8dc07faa8f9f 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/BUILD @@ -8,7 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/filter/http/compressor/v2:pkg", - "//envoy/extensions/compression/common/compressor/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto index 1f4ea2d65e49..1f6cd63e9d52 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.compressor.v3; import "envoy/config/core/v3/base.proto"; -import "envoy/extensions/compression/common/compressor/v3/compressor.proto"; +import "envoy/config/core/v3/extension.proto"; import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; @@ -56,5 +56,5 @@ message Compressor { // :ref:`envoy.filters.http.compressor.gzip` // is included in Envoy. // This field is ignored if used in the context of the gzip http-filter, but is mandatory otherwise. - compression.common.compressor.v3.CompressorLibrary compressor_library = 6; + config.core.v3.TypedExtensionConfig compressor_library = 6; } diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index 16593dbfd11a..24e974e012e5 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -10,7 +10,9 @@ CompressorFilterConfig::CompressorFilterConfig( const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime, Compression::Compressor::CompressorFactoryPtr compressor_factory) : Common::Compressors::CompressorFilterConfig( - generic_compressor, stats_prefix + "compressor." + compressor_factory->statsPrefix(), + generic_compressor, + stats_prefix + "compressor." + generic_compressor.compressor_library().name() + "." + + compressor_factory->statsPrefix(), scope, runtime, compressor_factory->contentEncoding()), compressor_factory_(std::move(compressor_factory)) {} diff --git a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc index 92c4ca5b3657..c43869f9bc8e 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_integration_test.cc @@ -75,6 +75,7 @@ class CompressorIntegrationTest : public testing::TestWithParam Date: Fri, 8 May 2020 12:40:23 +0300 Subject: [PATCH 56/56] reflect config changes in documentation Signed-off-by: Dmitry Rozhkov --- .../configuration/http/http_filters/compressor_filter.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst index 5bf7ef806ecb..08e7298e1dc6 100644 --- a/docs/root/configuration/http/http_filters/compressor_filter.rst +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -40,13 +40,13 @@ An example configuration of the filter may look like the following: - text/html - application/json compressor_library: - name: gzip + name: text_optimized typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.gzip.v3.Gzip memory_level: 3 window_bits: 10 compression_level: best - compression_strategy: rle + compression_strategy: default_strategy By *default* compression will be *skipped* when: @@ -86,7 +86,9 @@ When compression is *applied*: Statistics ---------- -Every configured Compressor filter has statistics rooted at ..* with the following: +Every configured Compressor filter has statistics rooted at +.compressor...* +with the following: .. csv-table:: :header: Name, Type, Description