Merge branch 'develop' into pyup-update-autopep8-1.5.7-to-1.6.0

.github/workflows/test-brew.yml

@@ -9,13 +9,13 @@ jobs:
     strategy:
       matrix:
         os: [macOS]
-        python: [3.8]
+        python: ['3.10']
       max-parallel: 1
       fail-fast: false
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
       - name: Brew

.github/workflows/test-dashboard.yml

@@ -9,13 +9,13 @@ jobs:
     strategy:
       matrix:
         os: [macOS, ubuntu, windows]
-        node: [10.x, 11.x, 12.x]
+        node: ['10.x', '11.x', '12.x']
       max-parallel: 4
       fail-fast: false
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Setup Node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node }}
       - name: Install Dependencies

.github/workflows/test-docker.yml

@@ -9,13 +9,13 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu]
-        python: [3.8]
+        python: ['3.10']
       max-parallel: 1
       fail-fast: false
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
       - name: Install dependencies

.github/workflows/test-library.yml

@@ -9,13 +9,13 @@ jobs:
     strategy:
       matrix:
         os: [macOS, ubuntu, windows]
-        python: [3.6, 3.7, 3.8]
+        python: ['3.6', '3.7', '3.8', '3.9', '3.10']
       max-parallel: 4
       fail-fast: false
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
       - name: Install dependencies

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-alpine as base
+FROM python:3.10-alpine as base
 FROM base as builder
 
 COPY requirements.txt /app/

README.md

@@ -20,11 +20,9 @@
 [![Contributions Welcome](https://img.shields.io/static/v1?label=contributions&message=welcome%20%F0%9F%91%8D&color=green)](https://github.com/abhinavsingh/proxy.py/issues)
 [![Gitter](https://badges.gitter.im/proxy-py/community.svg)](https://gitter.im/proxy-py/community)
 
-[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9&color=blue)](https://www.python.org/)
+[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue)](https://www.python.org/)
 [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue)](http://mypy-lang.org/)
 
-[![Become a Backer](https://opencollective.com/proxypy/tiers/backer.svg?avatarHeight=72)](https://opencollective.com/proxypy)
-
 # Table of Contents
 
 - [Features](#features)
@@ -94,6 +92,8 @@
   - [Public Key Infrastructure](#pki)
     - [API Usage](#api-usage)
     - [CLI Usage](#cli-usage)
+- [Run Dashboard](#run-dashboard)
+  - [Inspect Traffic](#inspect-traffic)
 - [Frequently Asked Questions](#frequently-asked-questions)
   - [Threads vs Threadless](#threads-vs-threadless)
   - [SyntaxError: invalid syntax](#syntaxerror-invalid-syntax)
@@ -1537,6 +1537,45 @@ FILE
     /Users/abhinav/Dev/proxy.py/proxy/__init__.py
 ```
 
+# Run Dashboard
+
+Dashboard is currently under development and not yet bundled with `pip` packages.  To run dashboard, you must checkout the source.
+
+Dashboard is written in Typescript and SCSS, so let's build it first using:
+
+```bash
+$ make dashboard
+```
+
+Now start `proxy.py` with dashboard plugin and by overriding root directory for static server:
+
+```bash
+$ proxy --enable-dashboard --static-server-dir dashboard/public
+...[redacted]... - Loaded plugin proxy.http.server.HttpWebServerPlugin
+...[redacted]... - Loaded plugin proxy.dashboard.dashboard.ProxyDashboard
+...[redacted]... - Loaded plugin proxy.dashboard.inspect_traffic.InspectTrafficPlugin
+...[redacted]... - Loaded plugin proxy.http.inspector.DevtoolsProtocolPlugin
+...[redacted]... - Loaded plugin proxy.http.proxy.HttpProxyPlugin
+...[redacted]... - Listening on ::1:8899
+...[redacted]... - Core Event enabled
+```
+
+Currently, enabling dashboard will also enable all the dashboard plugins.
+
+Visit dashboard:
+
+```bash
+$ open http://localhost:8899/dashboard/
+```
+
+## Inspect Traffic
+
+Wait for embedded `Chrome Dev Console` to load.  Currently, detail about all traffic flowing through `proxy.py` is pushed to the `Inspect Traffic` tab.  However, received payloads are not yet integrated with the embedded dev console.
+
+Current functionality can be verified by opening the `Dev Console` of dashboard and inspecting the websocket connection that dashboard established with the `proxy.py` server.
+
+[![Proxy.Py Dashboard Inspect Traffic](https://raw.githubusercontent.com/abhinavsingh/proxy.py/v3.4.0/Dashboard.png)](https://github.com/abhinavsingh/proxy.py)
+
 # Frequently Asked Questions
 
 ## Threads vs Threadless
@@ -1670,129 +1709,72 @@ few obvious ones include:
 
 ```bash
 ❯ proxy -h
-usage: proxy [-h] [--threadless] [--backlog BACKLOG] [--enable-events]
-          [--hostname HOSTNAME] [--port PORT] [--num-workers NUM_WORKERS]
-          [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE]
-          [--timeout TIMEOUT] [--pid-file PID_FILE] [--version]
-          [--disable-http-proxy] [--enable-dashboard] [--enable-devtools]
-          [--enable-static-server] [--enable-web-server]
-          [--log-level LOG_LEVEL] [--log-file LOG_FILE]
-          [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT]
-          [--plugins PLUGINS] [--ca-key-file CA_KEY_FILE]
-          [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE]
-          [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE]
-          [--cert-file CERT_FILE] [--disable-headers DISABLE_HEADERS]
-          [--server-recvbuf-size SERVER_RECVBUF_SIZE]
-          [--basic-auth BASIC_AUTH] [--cache-dir CACHE_DIR]
-          [--static-server-dir STATIC_SERVER_DIR] [--pac-file PAC_FILE]
-          [--pac-file-url-path PAC_FILE_URL_PATH]
-          [--filtered-client-ips FILTERED_CLIENT_IPS]
-
-proxy.py v2.3.1
+usage: proxy [-h] [--threadless] [--backlog BACKLOG] [--enable-events] [--hostname HOSTNAME] [--port PORT] [--num-workers NUM_WORKERS] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE]
+          [--timeout TIMEOUT] [--pid-file PID_FILE] [--version] [--disable-http-proxy] [--enable-dashboard] [--enable-devtools] [--enable-static-server] [--enable-web-server] [--log-level LOG_LEVEL]
+          [--log-file LOG_FILE] [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS] [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE]
+          [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE] [--disable-headers DISABLE_HEADERS] [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--basic-auth BASIC_AUTH]
+          [--cache-dir CACHE_DIR] [--static-server-dir STATIC_SERVER_DIR] [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH] [--filtered-client-ips FILTERED_CLIENT_IPS]
 
-optional arguments:
+proxy.py v2.4.0
+
+options:
   -h, --help            show this help message and exit
-  --threadless          Default: False. When disabled a new thread is spawned
-                        to handle each client connection.
-  --backlog BACKLOG     Default: 100. Maximum number of pending connections to
-                        proxy server
-  --enable-events       Default: False. Enables core to dispatch lifecycle
-                        events. Plugins can be used to subscribe for core
-                        events.
+  --threadless          Default: False. When disabled a new thread is spawned to handle each client connection.
+  --backlog BACKLOG     Default: 100. Maximum number of pending connections to proxy server
+  --enable-events       Default: False. Enables core to dispatch lifecycle events. Plugins can be used to subscribe for core events.
   --hostname HOSTNAME   Default: ::1. Server IP address.
   --port PORT           Default: 8899. Server port.
   --num-workers NUM_WORKERS
                         Defaults to number of CPU cores.
   --client-recvbuf-size CLIENT_RECVBUF_SIZE
-                        Default: 1 MB. Maximum amount of data received from
-                        the client in a single recv() operation. Bump this
-                        value for faster uploads at the expense of increased
-                        RAM.
-  --key-file KEY_FILE   Default: None. Server key file to enable end-to-end
-                        TLS encryption with clients. If used, must also pass
-                        --cert-file.
-  --timeout TIMEOUT     Default: 10. Number of seconds after which an inactive
-                        connection must be dropped. Inactivity is defined by
-                        no data sent or received by the client.
+                        Default: 1 MB. Maximum amount of data received from the client in a single recv() operation. Bump this value for faster uploads at the expense of increased RAM.
+  --key-file KEY_FILE   Default: None. Server key file to enable end-to-end TLS encryption with clients. If used, must also pass --cert-file.
+  --timeout TIMEOUT     Default: 10. Number of seconds after which an inactive connection must be dropped. Inactivity is defined by no data sent or received by the client.
   --pid-file PID_FILE   Default: None. Save parent process ID to a file.
   --version, -v         Prints proxy.py version.
-  --disable-http-proxy  Default: False. Whether to disable
-                        proxy.HttpProxyPlugin.
+  --disable-http-proxy  Default: False. Whether to disable proxy.HttpProxyPlugin.
   --enable-dashboard    Default: False. Enables proxy.py dashboard.
-  --enable-devtools     Default: False. Enables integration with Chrome
-                        Devtool Frontend. Also see --devtools-ws-path.
+  --enable-devtools     Default: False. Enables integration with Chrome Devtool Frontend. Also see --devtools-ws-path.
   --enable-static-server
-                        Default: False. Enable inbuilt static file server.
-                        Optionally, also use --static-server-dir to serve
-                        static content from custom directory. By default,
-                        static file server serves out of installed proxy.py
-                        python module folder.
-  --enable-web-server   Default: False. Whether to enable
-                        proxy.HttpWebServerPlugin.
+                        Default: False. Enable inbuilt static file server. Optionally, also use --static-server-dir to serve static content from custom directory. By default, static file server serves out of
+                        installed proxy.py python module folder.
+  --enable-web-server   Default: False. Whether to enable proxy.HttpWebServerPlugin.
   --log-level LOG_LEVEL
-                        Valid options: DEBUG, INFO (default), WARNING, ERROR,
-                        CRITICAL. Both upper and lowercase values are allowed.
-                        You may also simply use the leading character e.g.
-                        --log-level d
+                        Valid options: DEBUG, INFO (default), WARNING, ERROR, CRITICAL. Both upper and lowercase values are allowed. You may also simply use the leading character e.g. --log-level d
   --log-file LOG_FILE   Default: sys.stdout. Log file destination.
   --log-format LOG_FORMAT
                         Log format for Python logger.
   --open-file-limit OPEN_FILE_LIMIT
-                        Default: 1024. Maximum number of files (TCP
-                        connections) that proxy.py can open concurrently.
+                        Default: 1024. Maximum number of files (TCP connections) that proxy.py can open concurrently.
   --plugins PLUGINS     Comma separated plugins
   --ca-key-file CA_KEY_FILE
-                        Default: None. CA key to use for signing dynamically
-                        generated HTTPS certificates. If used, must also pass
-                        --ca-cert-file and --ca-signing-key-file
+                        Default: None. CA key to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-cert-file and --ca-signing-key-file
   --ca-cert-dir CA_CERT_DIR
-                        Default: ~/.proxy.py. Directory to store dynamically
-                        generated certificates. Also see --ca-key-file, --ca-
-                        cert-file and --ca-signing-key-file
+                        Default: ~/.proxy.py. Directory to store dynamically generated certificates. Also see --ca-key-file, --ca-cert-file and --ca-signing-key-file
   --ca-cert-file CA_CERT_FILE
-                        Default: None. Signing certificate to use for signing
-                        dynamically generated HTTPS certificates. If used,
-                        must also pass --ca-key-file and --ca-signing-key-file
-  --ca-file CA_FILE     Default: None. Provide path to custom CA file for peer
-                        certificate validation. Specially useful on MacOS.
+                        Default: None. Signing certificate to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file
+  --ca-file CA_FILE     Default: None. Provide path to custom CA file for peer certificate validation. Specially useful on MacOS.
   --ca-signing-key-file CA_SIGNING_KEY_FILE
-                        Default: None. CA signing key to use for dynamic
-                        generation of HTTPS certificates. If used, must also
-                        pass --ca-key-file and --ca-cert-file
+                        Default: None. CA signing key to use for dynamic generation of HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file
   --cert-file CERT_FILE
-                        Default: None. Server certificate to enable end-to-end
-                        TLS encryption with clients. If used, must also pass
-                        --key-file.
+                        Default: None. Server certificate to enable end-to-end TLS encryption with clients. If used, must also pass --key-file.
   --disable-headers DISABLE_HEADERS
-                        Default: None. Comma separated list of headers to
-                        remove before dispatching client request to upstream
-                        server.
+                        Default: None. Comma separated list of headers to remove before dispatching client request to upstream server.
   --server-recvbuf-size SERVER_RECVBUF_SIZE
-                        Default: 1 MB. Maximum amount of data received from
-                        the server in a single recv() operation. Bump this
-                        value for faster downloads at the expense of increased
-                        RAM.
+                        Default: 1 MB. Maximum amount of data received from the server in a single recv() operation. Bump this value for faster downloads at the expense of increased RAM.
   --basic-auth BASIC_AUTH
-                        Default: No authentication. Specify colon separated
-                        user:password to enable basic authentication.
+                        Default: No authentication. Specify colon separated user:password to enable basic authentication.
   --cache-dir CACHE_DIR
-                        Default: A temporary directory. Flag only applicable
-                        when cache plugin is used with on-disk storage.
+                        Default: A temporary directory. Flag only applicable when cache plugin is used with on-disk storage.
   --static-server-dir STATIC_SERVER_DIR
-                        Default: "public" folder in directory where proxy.py
-                        is placed. This option is only applicable when static
-                        server is also enabled. See --enable-static-server.
-  --pac-file PAC_FILE   A file (Proxy Auto Configuration) or string to serve
-                        when the server receives a direct file request. Using
-                        this option enables proxy.HttpWebServerPlugin.
+                        Default: "public" folder in directory where proxy.py is placed. This option is only applicable when static server is also enabled. See --enable-static-server.
+  --pac-file PAC_FILE   A file (Proxy Auto Configuration) or string to serve when the server receives a direct file request. Using this option enables proxy.HttpWebServerPlugin.
   --pac-file-url-path PAC_FILE_URL_PATH
                         Default: /. Web server path to serve the PAC file.
   --filtered-client-ips FILTERED_CLIENT_IPS
-                        Default: 127.0.0.1,::1. Comma separated list of IPv4
-                        and IPv6 addresses.
+                        Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6 addresses.
 
-Proxy.py not working? Report at:
-https://github.com/abhinavsingh/proxy.py/issues/new
+Proxy.py not working? Report at: https://github.com/abhinavsingh/proxy.py/issues/new
 ```
 
 # Changelog

proxy/common/version.py

@@ -8,5 +8,5 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-VERSION = (2, 3, 1)
+VERSION = (2, 4, 0)
 __version__ = '.'.join(map(str, VERSION[0:3]))

proxy/core/acceptor/pool.py

@@ -63,11 +63,10 @@
 
 
 class AcceptorPool:
-    """AcceptorPool.
-
-    Pre-spawns worker processes to utilize all cores available on the system.
+    """AcceptorPool pre-spawns worker processes to utilize all cores available on the system.
     A server socket is initialized and dispatched over a pipe to these workers.
-    Each worker process then accepts new client connection.
+    Each worker process then concurrently accepts new client connection over
+    the initialized server socket.
 
     Example usage:
 
@@ -83,7 +82,9 @@ class AcceptorPool:
 
     Optionally, AcceptorPool also initialize a global event queue.
     It is a multiprocess safe queue which can be used to build pubsub patterns
-    for message sharing or signaling within proxy.py.
+    for message sharing or signaling.
+
+    TODO(abhinavsingh): Decouple event queue setup & teardown into its own class.
     """
 
     def __init__(self, flags: argparse.Namespace,
@@ -110,9 +111,10 @@ def listen(self) -> None:
         self.socket.bind((str(self.flags.hostname), self.flags.port))
         self.socket.listen(self.flags.backlog)
         self.socket.setblocking(False)
-        logger.info(
-            'Listening on %s:%d' %
-            (self.flags.hostname, self.flags.port))
+        # Override flags.port to match the actual port
+        # we are listening upon.  This is necessary to preserve
+        # the server port when `--port=0` is used.
+        self.flags.port = self.socket.getsockname()[1]
 
     def start_workers(self) -> None:
         """Start worker processes."""
@@ -172,7 +174,6 @@ def setup(self) -> None:
             logger.info('Core Event enabled')
             self.start_event_dispatcher()
         self.start_workers()
-
         # Send server socket to all acceptor processes.
         assert self.socket is not None
         for index in range(self.flags.num_workers):