From ae5990230ace27c38c1e6a5e026e4f70a021a8bb Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 5 Jul 2024 15:47:18 +0100 Subject: [PATCH 1/4] Remove status_code addon Use callback addon instead. --- internal/deploy/mitm.go | 50 ++++++++++------- tests/mitmproxy_addons/README.md | 47 +++++++++++++++- tests/mitmproxy_addons/__init__.py | 2 - tests/mitmproxy_addons/callback.py | 1 + tests/mitmproxy_addons/status_code.py | 81 --------------------------- 5 files changed, 75 insertions(+), 106 deletions(-) delete mode 100644 tests/mitmproxy_addons/status_code.py diff --git a/internal/deploy/mitm.go b/internal/deploy/mitm.go index 971d187..d5b7ae4 100644 --- a/internal/deploy/mitm.go +++ b/internal/deploy/mitm.go @@ -8,6 +8,7 @@ import ( "net/url" "strings" "sync" + "sync/atomic" "testing" "time" @@ -111,39 +112,48 @@ func (c *MITMConfiguration) Execute(inner func()) { // $addon_values... // } // } - // We have 2 add-ons: "callback" and "status_code". The former just sniffs - // requests/responses. The latter modifies the request/response in some way. // // The API shape of the add-ons are located inside the python files in tests/mitmproxy_addons - body := map[string]any{} if len(c.pathCfgs) > 1 { c.t.Fatalf(">1 path config currently unsupported") // TODO } c.mu.Lock() + callbackAddon := map[string]any{} for _, pathConfig := range c.pathCfgs { + if pathConfig.filter() != "" { + callbackAddon["filter"] = pathConfig.filter() + } + cbServer, err := NewCallbackServer(c.t, c.client.hostnameRunningComplement) + must.NotError(c.t, "failed to start callback server", err) + defer cbServer.Close() + if pathConfig.listener != nil { - cbServer, err := NewCallbackServer(c.t, c.client.hostnameRunningComplement) - must.NotError(c.t, "failed to start callback server", err) - callbackURL := cbServer.SetOnResponseCallback(c.t, pathConfig.listener) - defer cbServer.Close() - - body["callback"] = map[string]any{ - "callback_response_url": callbackURL, - "filter": pathConfig.filter(), - } + responseCallbackURL := cbServer.SetOnResponseCallback(c.t, pathConfig.listener) + callbackAddon["callback_response_url"] = responseCallbackURL } - if pathConfig.blockRequest != nil { - body["statuscode"] = map[string]any{ - "return_status": pathConfig.blockStatusCode, - "block_request": *pathConfig.blockRequest, - "count": pathConfig.blockCount, - "filter": pathConfig.filter(), - } + if pathConfig.blockRequest != nil && *pathConfig.blockRequest { + // reimplement statuscode plugin logic in Go + // TODO: refactor this + var count atomic.Uint32 + requestCallbackURL := cbServer.SetOnRequestCallback(c.t, func(cd CallbackData) *CallbackResponse { + newCount := count.Add(1) + if pathConfig.blockCount > 0 && newCount > uint32(pathConfig.blockCount) { + return nil // don't block + } + // block this request by sending back a fake response + return &CallbackResponse{ + RespondStatusCode: pathConfig.blockStatusCode, + RespondBody: json.RawMessage(`{"error":"complement-crypto says no"}`), + } + }) + callbackAddon["callback_request_url"] = requestCallbackURL } } c.mu.Unlock() - lockID := c.client.lockOptions(c.t, body) + lockID := c.client.lockOptions(c.t, map[string]any{ + "callback": callbackAddon, + }) defer c.client.unlockOptions(c.t, lockID) inner() diff --git a/tests/mitmproxy_addons/README.md b/tests/mitmproxy_addons/README.md index 0550e01..34a85ba 100644 --- a/tests/mitmproxy_addons/README.md +++ b/tests/mitmproxy_addons/README.md @@ -11,8 +11,6 @@ How this works: ### Controller HTTP API -**This is highly experimental and will change without warning.** - `mitmproxy` is run once for all tests. To avoid test pollution, the controller is "locked" for the duration of a test and must be "unlocked" afterwards. When acquiring the lock, options can be set on `mitmproxy`. @@ -21,6 +19,9 @@ POST /options/lock { "options": { "body_size_limit": "3m", + "callback": { + "callback_response_url": "http://host.docker.internal:445566" + } } } HTTP/1.1 200 OK @@ -28,10 +29,50 @@ POST /options/lock "reset_id": "some_opaque_string" } ``` +Any [option](https://docs.mitmproxy.org/stable/concepts-options/) can be specified in the +`options` object, not just Complement specific addons. ``` POST /options/unlock { "reset_id": "some_opaque_string" } -``` \ No newline at end of file +``` + +### Callback addon + +A [mitmproxy addon](https://docs.mitmproxy.org/stable/addons-examples/) bolts on custom +functionality to mitmproxy. This typically involves using the +[Event Hooks API](https://docs.mitmproxy.org/stable/api/events.html) to listen for +[HTTP flows](https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#HTTPFlow). + +The `callback` addon is a Complement-Crypto specific addon which calls a client provided URL +mid-flow, with a JSON object containing information about the HTTP flow. The caller can then +return another JSON object which can modify the response in some way. + +Available configuration options: + - `callback_request_url` + - `callback_response_url` + - `filter` + +mitmproxy will call the callback URL with the following JSON object: +```js +{ + method: "GET|PUT|...", + access_token: "syt_11...", + url: "http://hs1/_matrix/client/...", + request_body: { some json object or null if no body }, + + // these fields will be missing for `callback_request_url` callbacks as the request + // has not yet been sent to the server. + response_body: { some json object }, + response_code: 200, +} +``` +The callback server can then return optional keys to replace parts of the response: +```js +{ + respond_status_code: 200, + respond_body: { "some": "json_object" } +} +``` diff --git a/tests/mitmproxy_addons/__init__.py b/tests/mitmproxy_addons/__init__.py index 91c4367..3a1be27 100644 --- a/tests/mitmproxy_addons/__init__.py +++ b/tests/mitmproxy_addons/__init__.py @@ -11,12 +11,10 @@ def install(package): install("aiohttp") from callback import Callback -from status_code import StatusCode from controller import MITM_DOMAIN_NAME, app addons = [ asgiapp.WSGIApp(app, MITM_DOMAIN_NAME, 80), # requests to this host will be routed to the flask app - StatusCode(), Callback(), ] # testcontainers will look for this log line diff --git a/tests/mitmproxy_addons/callback.py b/tests/mitmproxy_addons/callback.py index f553a7c..b115068 100644 --- a/tests/mitmproxy_addons/callback.py +++ b/tests/mitmproxy_addons/callback.py @@ -149,6 +149,7 @@ async def send_callback(self, flow, url: str, body: dict): # For responses: fields are optional but the default case is always specified. respond_status_code = test_response_body.get("respond_status_code", body.get("response_code")) respond_body = test_response_body.get("respond_body", body.get("response_body")) + print(f'{datetime.now().strftime("%H:%M:%S.%f")} callback for {flow.request.url} returning custom response: HTTP {respond_status_code} {json.dumps(respond_body)}') flow.response = Response.make( respond_status_code, json.dumps(respond_body), headers={ diff --git a/tests/mitmproxy_addons/status_code.py b/tests/mitmproxy_addons/status_code.py deleted file mode 100644 index aceb791..0000000 --- a/tests/mitmproxy_addons/status_code.py +++ /dev/null @@ -1,81 +0,0 @@ -from typing import Optional - -from mitmproxy import ctx, flowfilter -from mitmproxy.http import Response -from controller import MITM_DOMAIN_NAME - -# StatusCode will intercept a response and return the provided status code in its place, with -# no response body. Supports filters: https://docs.mitmproxy.org/stable/concepts-filters/ -class StatusCode: - def __init__(self): - self.reset() - print(MITM_DOMAIN_NAME) - self.matchall = flowfilter.parse(".") - self.filter: Optional[flowfilter.TFilter] = self.matchall - self._seen_count = 0 - - def reset(self): - self.config = { - "return_status": 0, - "block_request": False, - "count": 0, - "filter": None, - } - self._seen_count = 0 - - def exceeded_count(self): - return self.config["count"] > 0 and self._seen_count >= self.config["count"] - - def load(self, loader): - loader.add_option( - name="statuscode", - typespec=dict, - default={"return_status": 0, "filter": None, "block_request": False, "count": 0}, - help="Change the response status code, with an optional filter", - ) - - def configure(self, updates): - if "statuscode" not in updates: - self.reset() - return - if ctx.options.statuscode is None or ctx.options.statuscode["return_status"] == 0: - self.reset() - return - self.config = ctx.options.statuscode - # ensure optional fields are set - self.config.setdefault("count", 0) - self.config.setdefault("block_request", False) - - new_filter = self.config.get('filter', None) - print(f"statuscode will return HTTP {self.config['return_status']} filter={new_filter} count={self.config['count']}") - if new_filter: - self.filter = flowfilter.parse(new_filter) - else: - self.filter = self.matchall - - def request(self, flow): - # always ignore the controller - if flow.request.pretty_host == MITM_DOMAIN_NAME: - return - if self.config["return_status"] == 0: - return # ignore responses if we aren't told a code - if self.config["block_request"] and flowfilter.match(self.filter, flow) and not self.exceeded_count(): - self._seen_count += 1 - print(f'statuscode: blocking request and sending back {self.config["return_status"]}: count {self._seen_count}/{self.config["count"]}') - flow.response = Response.make(self.config["return_status"], headers={ - "MITM-Proxy": "yes", - }) - - def response(self, flow): - # always ignore the controller - if flow.request.pretty_host == MITM_DOMAIN_NAME: - return - if self.config["return_status"] == 0: - return # ignore responses if we aren't told a code - if flow.response.headers.get("MITM-Proxy", None) is not None: - print(f'not modifying mitm response!') - return # ignore responses generated by mitm proxy (i.e the one in `def request` above - if flowfilter.match(self.filter, flow) and not self.exceeded_count(): - self._seen_count += 1 - print(f'statuscode: blocking response and sending back {self.config["return_status"]}: count {self._seen_count}/{self.config["count"]}') - flow.response = Response.make(self.config["return_status"]) From 35f209122547bd88124f09c3f290dee6bf060dc7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 5 Jul 2024 15:59:29 +0100 Subject: [PATCH 2/4] Update docs --- tests/mitmproxy_addons/README.md | 58 ++++++++++++++++++++++++------ tests/mitmproxy_addons/callback.py | 30 +--------------- 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/tests/mitmproxy_addons/README.md b/tests/mitmproxy_addons/README.md index 34a85ba..bb54690 100644 --- a/tests/mitmproxy_addons/README.md +++ b/tests/mitmproxy_addons/README.md @@ -8,7 +8,6 @@ How this works: - It is also told to run a normal proxy, to which a Flask HTTP server is attached. - The Flask HTTP server can be used to control mitmproxy at test runtime. This is done via the Controller HTTP API. - ### Controller HTTP API `mitmproxy` is run once for all tests. To avoid test pollution, the controller is "locked" for the duration @@ -39,6 +38,8 @@ POST /options/unlock } ``` +Tests will lock/unlock whenever they need to interact with mitmproxy. + ### Callback addon A [mitmproxy addon](https://docs.mitmproxy.org/stable/addons-examples/) bolts on custom @@ -50,29 +51,66 @@ The `callback` addon is a Complement-Crypto specific addon which calls a client mid-flow, with a JSON object containing information about the HTTP flow. The caller can then return another JSON object which can modify the response in some way. -Available configuration options: - - `callback_request_url` - - `callback_response_url` - - `filter` +Available configuration options are optional: + - `callback_request_url`: the URL to send outbound requests to. This allows callbacks to intercept + requests BEFORE they reach the server. + - `callback_response_url`: the URL to send inbound responses to. This allows callbacks to modify + response content. + - `filter`: the mitmproxy filter to apply. If unset, ALL requests are eligible to go to the callback + server. + +To use this with the controller API, you would send an HTTP request like this: +```js +{ + "options": { + "callback": { + "callback_response_url": "http://host.docker.internal:445566/response" + } + } +} +``` -mitmproxy will call the callback URL with the following JSON object: +#### `callback_response_url` +mitmproxy will POST to `callback_response_url` with the following JSON object: ```js { method: "GET|PUT|...", access_token: "syt_11...", url: "http://hs1/_matrix/client/...", request_body: { some json object or null if no body }, - - // these fields will be missing for `callback_request_url` callbacks as the request - // has not yet been sent to the server. response_body: { some json object }, response_code: 200, } ``` -The callback server can then return optional keys to replace parts of the response: +The callback server can then return optional keys to replace parts of the response. +The values returned here will be returned to the Matrix client: +```js +{ + respond_status_code: 200, + respond_body: { "some": "json_object" } +} +``` +These keys are optional. If neither are specified, the response is sent unaltered to +the Matrix client. If the body is set but the status code is not, only the body is +modified and the status code is left unaltered and vice versa. + +#### `callback_request_url` + +Similarly, mitmproxy will POST to `callback_request_url` with the following JSON object: +```js +{ + method: "GET|PUT|...", + access_token: "syt_11...", + url: "http://hs1/_matrix/client/...", + request_body: { some json object or null if no body }, + // note the absence of response_* fields, as the request has yet to reach the HS +} +``` +The callback server can then either return an empty object or the following object (all fields are required): ```js { respond_status_code: 200, respond_body: { "some": "json_object" } } ``` +If an empty object is returned, mitmproxy will forward the request unaltered to the server. If the above object is returned, mitmproxy will send that response _immediately_ and **will not send the request to the server**. This can be used to block HTTP requests. \ No newline at end of file diff --git a/tests/mitmproxy_addons/callback.py b/tests/mitmproxy_addons/callback.py index b115068..d3000ee 100644 --- a/tests/mitmproxy_addons/callback.py +++ b/tests/mitmproxy_addons/callback.py @@ -10,35 +10,7 @@ from urllib.error import HTTPError, URLError from datetime import datetime -# Callback will intercept a request and/or response and send a POST request to the provided url, with -# the following JSON object. Supports filters: https://docs.mitmproxy.org/stable/concepts-filters/ -# { -# method: "GET|PUT|...", -# access_token: "syt_11...", -# url: "http://hs1/_matrix/client/...", -# request_body: { some json object or null if no body }, -# response_body: { some json object }, -# response_code: 200, -# } -# The response to this request can control what gets returned to the client. The response object: -# { -# "respond_status_code": 200, -# "respond_body": { "some": "json_object" } -# } -# If {} is sent back, the response is not modified. Likewise, if `respond_body` is set but -# `respond_status_code` is not, only the response body is modified, not the status code, and vice versa. -# -# To use this addon, configure it with these fields: -# - callback_request_url: the URL to send outbound requests to. This allows callbacks to intercept -# requests BEFORE they reach the server. The request/response struct in this -# callback is the same as `callback_response_url`, except `response_body` -# and `response_code` will be missing as the request hasn't been processed -# yet. To block the request from reaching the server, the callback needs to -# provide both `respond_status_code` and `respond_body`. -# - callback_response_url: the URL to send inbound responses to. This allows callbacks to modify -# response content. -# - filter: the mitmproxy filter to apply. If unset, ALL requests are eligible to go to the callback -# server. +# See README.md for information about this addon class Callback: def __init__(self): self.reset() From a4b0f9c90f93b6a85b4d0b4e681b08d277fb415e Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:05:41 +0100 Subject: [PATCH 3/4] More docs --- tests/mitmproxy_addons/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mitmproxy_addons/README.md b/tests/mitmproxy_addons/README.md index bb54690..2bb2d9d 100644 --- a/tests/mitmproxy_addons/README.md +++ b/tests/mitmproxy_addons/README.md @@ -38,7 +38,7 @@ POST /options/unlock } ``` -Tests will lock/unlock whenever they need to interact with mitmproxy. +Tests will lock/unlock whenever they need to interact with mitmproxy. Attempting to lock an already locked controller will return an HTTP 400 error. Attempting to unlock an already unlocked controller will return an HTTP 400 error. ### Callback addon @@ -56,7 +56,7 @@ Available configuration options are optional: requests BEFORE they reach the server. - `callback_response_url`: the URL to send inbound responses to. This allows callbacks to modify response content. - - `filter`: the mitmproxy filter to apply. If unset, ALL requests are eligible to go to the callback + - `filter`: the [mitmproxy filter](https://docs.mitmproxy.org/stable/concepts-filters/) to apply. If unset, ALL requests are eligible to go to the callback server. To use this with the controller API, you would send an HTTP request like this: From 2c78f48c188cc69e88be3149d1f1bf01206733c4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal <7190048+kegsay@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:17:06 +0100 Subject: [PATCH 4/4] Rejig order of urls --- tests/mitmproxy_addons/README.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/mitmproxy_addons/README.md b/tests/mitmproxy_addons/README.md index 2bb2d9d..d9c81f9 100644 --- a/tests/mitmproxy_addons/README.md +++ b/tests/mitmproxy_addons/README.md @@ -70,47 +70,48 @@ To use this with the controller API, you would send an HTTP request like this: } ``` -#### `callback_response_url` -mitmproxy will POST to `callback_response_url` with the following JSON object: +#### `callback_request_url` + +mitmproxy will POST to `callback_request_url` with the following JSON object: ```js { method: "GET|PUT|...", access_token: "syt_11...", url: "http://hs1/_matrix/client/...", request_body: { some json object or null if no body }, - response_body: { some json object }, - response_code: 200, } ``` -The callback server can then return optional keys to replace parts of the response. -The values returned here will be returned to the Matrix client: +The callback server can then either return an empty object or the following object (all fields are required): ```js { respond_status_code: 200, respond_body: { "some": "json_object" } } ``` -These keys are optional. If neither are specified, the response is sent unaltered to -the Matrix client. If the body is set but the status code is not, only the body is -modified and the status code is left unaltered and vice versa. +If an empty object is returned, mitmproxy will forward the request unaltered to the server. If the above object (with all fields set) is returned, mitmproxy will send that response _immediately_ and **will not send the request to the server**. This can be used to block HTTP requests. -#### `callback_request_url` -Similarly, mitmproxy will POST to `callback_request_url` with the following JSON object: +#### `callback_response_url` +Similarly, mitmproxy will POST to `callback_response_url` with the following JSON object: ```js { method: "GET|PUT|...", access_token: "syt_11...", url: "http://hs1/_matrix/client/...", request_body: { some json object or null if no body }, - // note the absence of response_* fields, as the request has yet to reach the HS + // note these are new fields because the request was sent to the HS and a response returned from it + response_body: { some json object }, + response_code: 200, } ``` -The callback server can then either return an empty object or the following object (all fields are required): +The callback server can then return optional keys to replace parts of the response. +The values returned here will be returned to the Matrix client: ```js { respond_status_code: 200, respond_body: { "some": "json_object" } } ``` -If an empty object is returned, mitmproxy will forward the request unaltered to the server. If the above object is returned, mitmproxy will send that response _immediately_ and **will not send the request to the server**. This can be used to block HTTP requests. \ No newline at end of file +These keys are optional. If neither are specified, the response is sent unaltered to +the Matrix client. If the body is set but the status code is not, only the body is +modified and the status code is left unaltered and vice versa. \ No newline at end of file