Skip to content

Commit

Permalink
Add upstream request/response validation with openapi (#21) (#22)
Browse files Browse the repository at this point in the history 
* upstream validation: implementation, first try (#21)

* add upstream swagger file, referenced in couper.hcl (#21)

* rename swagger_definition -> openapi_file (#21)

* openapi block with properties instead of properties only; request/response are always validated if openapi block is present; request is rejected if invalid and ignore_request_violations is not true; response is rejected if invalid and ignore_response_violations is not true (#21)

* invalid upstream request gets 400 - Bad Request; invalid upstream response gets 502 - Bad Gateway (#21)

* keep backend status code in log even in case of response validation error (#21)

* validation message in backend log's message field (#21)

* tests for validation (#21)

* extracted OpenAPI validator (#21)

* Fixed validation errors in openapi file to be logged as errors instead of panic (#21)

* Fixup openAPI validation test

* Handle body rewind

Refactor openAPI error handling

* Fix set getBody method first #72

Leads to getAttribute errors for recently added dynamic evals for origin, path and hostname

* Add buffer stringer implementation

Add stringer tool to generate for const

* Add test for bufferOption interaction

* rm httpbin.yaml

* Fix documentation hcl format

* Remove loose punctuation mark from documentation

Fix couper version to latest release

* Add openapi documentation and example link

* Fixup obsolete conditions

* Add validation exclude options

Fix passing the query param

* Fixup validation tests

e.g. query 404 -> 404= due to our set query feature

* Update validation documentation

* Use req context

* Fix merge openAPI and use partialContent for deprecated log

* Upgrade kin-openapi dependency to latest v0.33.0

* Add documentation note about openapi3

* Add additional openapi test

Remove own getBody set since openapi3 does this already (too)

* Revert configurable validation exclude options

* Add additonal openapi link

Co-authored-by: Marcel Ludwig <marcel.ludwig@avenga.com>
  • Loading branch information
@johakoch
johakoch and Marcel Ludwig authored Dec 14, 2020
1 parent 45b5dd0 commit 8feffb2
Show file tree
Hide file tree
Showing 230 changed files with 22,308 additions and 8,478 deletions.
5 changes: 5 additions & 0 deletions config/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type Backend struct {
RequestBodyLimit string `hcl:"request_body_limit,optional"`
TTFBTimeout string `hcl:"ttfb_timeout,optional"`
Timeout string `hcl:"timeout,optional"`
OpenAPI *OpenAPI `hcl:"openapi,block"`
}

func (b Backend) Body() hcl.Body {
Expand Down Expand Up @@ -83,6 +84,10 @@ func (b *Backend) Merge(other *Backend) (*Backend, []hcl.Body) {
result.Timeout = other.Timeout
}

if other.OpenAPI != nil {
result.OpenAPI = other.OpenAPI
}

return &result, bodies
}

Expand Down
7 changes: 7 additions & 0 deletions config/openapi.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package config

type OpenAPI struct {
File string `hcl:"file"`
IgnoreRequestViolations bool `hcl:"ignore_request_violations,optional"`
IgnoreResponseViolations bool `hcl:"ignore_response_violations,optional"`
}
11 changes: 6 additions & 5 deletions config/runtime/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
"github.com/avenga/couper/utils"
)

var defaultBackendConf = &config.Backend{
var DefaultBackendConf = &config.Backend{
ConnectTimeout: "10s",
RequestBodyLimit: "64MiB",
TTFBTimeout: "60s",
Expand Down Expand Up @@ -152,6 +152,7 @@ func NewServerConfiguration(conf *config.Gateway, httpConf *HTTPConfig, log *log
if srvConf.API != nil {
// map backends to endpoint
endpoints := make(map[string]bool)

for _, endpoint := range srvConf.API.Endpoint {
pattern := utils.JoinPath("/", serverOptions.APIBasePath, endpoint.Pattern)

Expand Down Expand Up @@ -215,11 +216,11 @@ func newProxy(ctx *hcl.EvalContext, beConf *config.Backend, corsOpts *config.COR

for _, name := range []string{"request_headers", "response_headers"} {
for _, body := range remainCtx {
attr, err := body.JustAttributes()
content, _, err := body.PartialContent(config.Backend{}.Schema(true))
if err != nil {
return nil, err
}
if _, ok := attr[name]; ok {
if _, ok := content.Attributes[name]; ok {
log.Warningf("'%s' is deprecated, use 'set_%s' instead", name, name)
}
}
Expand Down Expand Up @@ -250,7 +251,7 @@ func newBackendsFromDefinitions(conf *config.Gateway, confCtx *hcl.EvalContext,
return nil, e
}

beConf, _ = defaultBackendConf.Merge(beConf)
beConf, _ = DefaultBackendConf.Merge(beConf)

srvOpts, _ := server.NewServerOptions(&config.Server{})
proxy, err := newProxy(confCtx, beConf, nil, []hcl.Body{beConf.Remain}, log, srvOpts)
Expand Down Expand Up @@ -445,7 +446,7 @@ func newInlineBackend(
bodies = append(bodies, backendConf.Body())
}

backendConf, _ = defaultBackendConf.Merge(backendConf)
backendConf, _ = DefaultBackendConf.Merge(backendConf)

// obtain the backend reference and merge with the current override
if inlineBlock != nil && len(inlineBlock.Labels) > 0 {
Expand Down
246 changes: 134 additions & 112 deletions docs/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,38 @@
# Couper Docs - Version 0.1
# Couper Docs - Version 0.2

## Table of contents

* [Introduction](#introduction)
* [Core concepts](#core_concepts)
* [Configuration file](#conf_file)
* [Syntax](#syntax)
* [File name](#file_name)
* [Basic file structure](#basic_conf)
* [Variables](#variables_conf)
* [Expressions](#expressions)
* [Functions](#functions)
* [Core concepts](#core_concepts)
* [Configuration file](#conf_file)
* [Syntax](#syntax)
* [File name](#file_name)
* [Basic file structure](#basic_conf)
* [Variables](#variables_conf)
* [Expressions](#expressions)
* [Functions](#functions)
* [Reference](#reference)
* [The `server` block](#server_block)
* [The `files` block](#files_block)
* [The `spa` block](#spa_block)
* [The `api` block](#api_block)
* [The `endpoint` block](#endpoint_block)
* [The `backend` block](#backend_block)
* [The `request` block](#request_block)
* [The `cors` block](#cors_block)
* [The `access_control` attribute](#access_control_attribute)
* [The `basic_auth` block](#basic_auth_block)
* [The `jwt` block](#jwt_block)
* [The `definitions` block](#definitions_block)
* [The `defaults` block](#defaults_block)
* [The `settings` block](#settings_block)
* [The `server` block](#server_block)
* [The `files` block](#files_block)
* [The `spa` block](#spa_block)
* [The `api` block](#api_block)
* [The `endpoint` block](#endpoint_block)
* [The `backend` block](#backend_block)
* [The `openapi` block](#openapi_block)
* [The `cors` block](#cors_block)
* [The `request` block](#request_block)
* [The `access_control` attribute](#access_control_attribute)
* [The `definitions` block](#definitions_block)
* [The `basic_auth` block](#basic_auth_block)
* [The `jwt` block](#jwt_block)
* [The `defaults` block](#defaults_block)
* [The `settings` block](#settings_block)
* [Examples](#examples)
* [Request routing](#request_routing_ex)
* [Routing configuration](#routing_conf_ex)
* [Web serving configuration](#web_serving_ex)
* [`access_control`configuration](#access_control_conf_ex)
* [`hosts` configuration](#hosts_conf_ex)
* [Request routing](#request_routing_ex)
* [Routing configuration](#routing_conf_ex)
* [Web serving configuration](#web_serving_ex)
* [`access_control`configuration](#access_control_conf_ex)
* [`hosts` configuration](#hosts_conf_ex)

## Introduction <a name="introduction"></a>
Couper is a frontend gateway especially designed to support building and running API-driven Web projects.
Expand Down Expand Up @@ -73,27 +74,31 @@ For orientation compare the following example and the information below:

```hcl
server "my_project" {
files {...}
spa {...}
api {
access_control = "foo"
endpoint "/bar" {
backend {...}
}
}
definitions {...}
files { ... }
spa { ... }
api {
access_control = "foo"
endpoint "/bar" {
backend { ... }
}
}
}
definitions { ... }
```

* `server`: main configuration block
* `files`: configuration block for file serving
* `spa`: configuration block for web serving (spa assets)
* `api`: configuration block that bundles endpoints under a certain base path
* `access_control`: attribute that sets access control for a block context
* `endpoint`: configuration block for Couper's entry points
* `backend`: configuration block for connection to local/remote backend service(s)
* `definitions`: block for predefined configurations, that can be referenced
* `defaults`: block for default configurations
* `settings`: block for server configuration which applies to the running instance
* `server` main configuration block
* `files` configuration block for file serving
* `spa` configuration block for web serving (spa assets)
* `api` configuration block that bundles endpoints under a certain base path
* `access_control` attribute that sets access control for a block context
* `endpoint` configuration block for Couper's entry points
* `backend` configuration block for connection to local/remote backend service(s)
* `definitions` block for predefined configurations, that can be referenced
* `defaults` block for default configurations
* `settings` block for server configuration which applies to the running instance

### Variables <a name="variables_conf"></a>

Expand Down Expand Up @@ -334,19 +339,20 @@ A `backend` defines the connection to a local/remote backend service. Backends c

| Name | Description |
|:-------------------|:---------------------------------------|
|context|<ul><li>`api` block</li><li>`endpoint` block</li><li>`definitions` block (reference purpose)</li></ul>|
| context|<ul><li>`api` block</li><li>`endpoint` block</li><li>`definitions` block (reference purpose)</li></ul>|
| *label*|<ul><li>&#9888; mandatory, when declared in `api` block</li><li>&#9888; mandatory, when declared in `definitions` block</li></ul>|
| `origin`| URL to connect to for backend requests </br> &#9888; must start with `http://...` |
|`base_path`|<ul><li>`base_path` for backend</li><li>won\`t change for `endpoint`</li></ul> |
|`hostname`| value of the HTTP host header field for the `origin` request. Since `hostname` replaces the request host the value will also be used for a server identity check during a TLS handshake with the origin. |
|`path`|changeable part of upstream URL|
|`timeout`| <ul><li>the total deadline duration a backend request has for write and read/pipe</li><li>valid time units are: "ns", "us" (or "µs"), "ms", "s", "m", "h"</li></ul> |
| `base_path`|<ul><li>`base_path` for backend</li><li>won\`t change for `endpoint`</li></ul> |
| `hostname`| value of the HTTP host header field for the `origin` request. Since `hostname` replaces the request host the value will also be used for a server identity check during a TLS handshake with the origin. |
| `path`|changeable part of upstream URL|
| `timeout`| <ul><li>the total deadline duration a backend request has for write and read/pipe</li><li>valid time units are: "ns", "us" (or "µs"), "ms", "s", "m", "h"</li></ul> |
| `set_request_headers` | header map to define additional or override header for the `origin` request |
| `set_response_headers` | same as `set_request_headers` for the client response |
| `request_body_limit` | Limit to configure the maximum buffer size while accessing `req.post` or `req.json_body` content. Valid units are: `KiB, MiB, GiB`. Default: `64MiB`. |
|[**`remove_query_params`**](#query_params)|<ul><li>a list of query parameters to be removed from the upstream request URL</li></ul> |
|[**`set_query_params`**](#query_params)|<ul><li>key/value(s) pairs to set query parameters in the upstream request URL</li></ul> |
|[**`add_query_params`**](#query_params)|<ul><li>key/value(s) pairs to add query parameters to the upstream request URL</li></ul> |
| [`openapi`](#openapi_block) | Definition for validating outgoing requests to the `origin` and incoming responses from the `origin`. |
| [`remove_query_params`](#query_params)|<ul><li>a list of query parameters to be removed from the upstream request URL</li></ul> |
| [`set_query_params`](#query_params)|<ul><li>key/value(s) pairs to set query parameters in the upstream request URL</li></ul> |
| [`add_query_params`](#query_params)|<ul><li>key/value(s) pairs to add query parameters to the upstream request URL</li></ul> |

### The `access_control` attribute <a name="access_control_attribute"></a>
The configuration of access control is twofold in Couper: You define the particular type (such as `jwt` or `basic_auth`) in `definitions`, each with a distinct label. Anywhere in the `server` block those labels can be used in the `access_control` list to protect that block.
Expand Down Expand Up @@ -384,6 +390,22 @@ The `jwt` block let you configure JSON Web Token access control for your gateway
|`signature_algorithm`| valid values are: `RS256` `RS384` `RS512` `HS256` `HS384` `HS512` |
|**`claims`**|equals/in comparison with JWT payload|

#### The `openapi` block <a name="openapi_block"></a>
The `openapi` block configures the backends proxy behaviour to validate outgoing and incoming requests to and from the origin.
Preventing the origin from invalid requests, and the Couper client from invalid answers. An example can be found [here](https://github.com/avenga/couper-examples/blob/master/backend-validation/README.md).
To do so Couper uses the [OpenAPI 3 standard](https://www.openapis.org/) to load the definitions from a given document
defined with the `file` attribute.

| Name | Description | Default |
|:-----------------------------|:---------------------------------------------------|:----------|
| context | `backend` block | |
| `file` | OpenAPI yaml definition file | mandatory |
| `ignore_request_violations` | log request validation results, skip err handling | `false` |
| `ignore_response_violations` | log response validation results, skip err handling | `false` |

**Caveats**: While ignoring request violations an invalid method or path would lead to a non-matching *route* which is still required
for response validations. In this case the response validation will fail if not ignored too.

### The `definitions` block <a name="definitions_block"></a>
Use the `definitions` block to define configurations you want to reuse. `access_control` is **always** defined in the `definitions` block.

Expand Down Expand Up @@ -425,84 +447,84 @@ The shutdown timings cannot be configured at this moment.

```hcl
api "my_api" {
base_path = "/api/novoconnect"
endpoint "/login/**" {
# incoming request: .../login/foo
# implicit proxy
# outgoing request: http://identityprovider:8080/login/foo
backend {
origin = "http://identityprovider:8080"
}
base_path = "/api/novoconnect"
endpoint "/login/**" {
# incoming request: .../login/foo
# implicit proxy
# outgoing request: http://identityprovider:8080/login/foo
backend {
origin = "http://identityprovider:8080"
}
endpoint "/cart/**" {
}
endpoint "/cart/**" {
# incoming request: .../cart/items
# outgoing request: http://cartservice:8080/api/v1/items
path = "/api/v1/**"
backend {
origin = "http://cartservice:8080"
}
endpoint "/account/{id}" {
# incoming request: .../account/brenda
# outgoing request: http://accountservice:8080/user/brenda/info
backend {
path = "/user/${req.param.id}/info"
origin = "http://accountservice:8080"
}
endpoint "/account/{id}" {
# incoming request: .../account/brenda
# outgoing request: http://accountservice:8080/user/brenda/info
backend {
path = "/user/${req.param.id}/info"
origin = "http://accountservice:8080"
}
}
}
}
```

### Web serving configuration example <a name="web_serving_ex"></a>
```hcl
server "my_project" {
files {
document_root = "./htdocs"
error_file = "./my_custom_error_page.html"
}
spa {
bootstrap_file = "./htdocs/index.html"
paths = [
"/app/**",
"/profile/**"
]
}
...
files {
document_root = "./htdocs"
error_file = "./my_custom_error_page.html"
}
spa {
bootstrap_file = "./htdocs/index.html"
paths = [
"/app/**",
"/profile/**"
]
}
}
```

### `access_control` configuration example <a name="access_control_conf_ex"></a>

```hcl
server {
access\_control = ["ac1"]
files {
access\_control = ["ac2"]
}
spa {}
api {
access\_control = ["ac3"]
endpoint "/foo" {
disable\_access_control = "ac3"
}
endpoint "/bar" {
access\_control = ["ac4"]
}
}
access_control = ["ac1"]
files {
access_control = ["ac2"]
}
spa {
bootstrap_file = "myapp.html"
}
api {
access_control = ["ac3"]
endpoint "/foo" {
disable_access_control = "ac3"
}
endpoint "/bar" {
access_control = ["ac4"]
}
}
}
definitions {
basic\_auth "ac1" {
...
}
jwt "ac2" {
...
}
jwt "ac3" {
...
}
jwt "ac4" {
...
}
basic_auth "ac1" { ... }
jwt "ac2" { ... }
jwt "ac3" { ... }
jwt "ac4" { ... }
}
```

Expand Down
Loading

0 comments on commit 8feffb2

Please sign in to comment.