initial commit for working transparent https proxy setup using TLS-na…

…tive proxy with SNI-based forwarding

tests/transparent-setup/3proxy-CONNECT.cfg

@@ -1,3 +1,9 @@
+# 3proxy-TLS.cfg replaces this config given that SNI is being used. If SNI is not used or a potentially very old ssl (pre TLS) version this config could be used
+# this config forwards transparently redirected (to port 8012) TCP connections to original destination IPs and port via parent upstream proxy
+# problem is that using this setup the upstream proxy sees the IP as target authority in CONNECT method instead of the SNI hostname https://github.com/3proxy/3proxy/issues/1084
+# as described in the issue, RFC 7231 states that this is not allowed for proxied https requests 
+# this also leads to issues with selective TLS interception in proxypy (httpsinterception in archivo mode) as reported in https://github.com/abhinavsingh/proxy.py/issues/1493
+
 plugin /usr/local/3proxy/libexec/TransparentPlugin.ld.so transparent_plugin
 log
 logformat "L[%Y-%m-%dT%H:%M:%S.%.]_HTTPS/CONNECT-proxy || %U@%N:%p src: %C:%c dst: %R:%r orig_dst: %Q:%q ext_iface: %e hops: %h host: %n bytes-I/0: %I/%O byte/sec-I/O: %B/%b duration: %D text: %T error: %E"
@@ -8,5 +14,5 @@ allow *
 parent 1000 connect+ tools.dbpedia.org 8894 user1 mypassword
 
 transparent
-tcppm -i0.0.0.0 8012 127.0.0.1 11111
-#notransparent
+tcppm -i0.0.0.0 8013 127.0.0.1 11111
+# NOTE the IP adress and port after tcppm is irrelevant, it is just a placeholder and will be replaced by original destination IP and port due to transparent keyword 

tests/transparent-setup/3proxy-Dockerfile-full

@@ -0,0 +1,66 @@
+# 3proxy.full is fully functional 3proxy build based on busibox:glibc
+#
+#to  build:
+# docker build -f Dockerfile.full -t 3proxy.full .
+#to run:
+# by default 3proxy uses safe chroot environment with chroot to /usr/local/3proxy with uid/gid 65535/65535 and expects
+# configuration file to be placed in /usr/local/etc/3proxy.
+# Paths in configuration file must be relative to /usr/local/3proxy, that is use /logs instead of 
+# /usr/local/3proxy/logs. nserver in chroot is required for DNS resolution. An example:
+#
+# echo nserver 8.8.8.8 >/path/to/local/config/directory/3proxy.cfg
+# echo proxy -p3129 >>/path/to/local/config/directory/3proxy.cfg
+# docker run -p 3129:3129 -v /path/to/local/config/directory:/usr/local/3proxy/conf -name 3proxy.full 3proxy.full
+#
+# /path/to/local/config/directory in this example must conrain 3proxy.cfg
+# if you need 3proxy to be executed without chroot with root permissions, replace /etc/3proxy/3proxy.cfg by e.g. mounting config
+# dir to /etc/3proxy ot by providing config file /etc/3proxy/3proxy.cfg
+# docker run -p 3129:3129 -v /path/to/local/config/directory:/etc/3proxy -name 3proxy.full 3proxy.full
+#
+# use "log" without pathname in config to log to stdout.
+# plugins are located in /usr/local/3proxy/libexec (/libexec for chroot config).
+
+# switched to ubuntu:20.04 to not have problems with apt install on ubuntu 20.04 hosts
+FROM ubuntu:20.04 AS buildenv 
+# COPY . 3proxyt
+RUN apt-get update && apt-get install -y git gcc build-essential libssl-dev tree
+RUN git clone https://github.com/3proxy/3proxy
+RUN cd 3proxy &&\
+ echo "">> Makefile.Linux &&\
+ echo PLUGINS = StringsPlugin TrafficPlugin PCREPlugin TransparentPlugin SSLPlugin>>Makefile.Linux &&\
+ echo LIBS = -l:libcrypto.a -l:libssl.a -ldl >>Makefile.Linux &&\
+ make -f Makefile.Linux &&\
+ strip bin/3proxy &&\
+ strip bin/StringsPlugin.ld.so &&\
+ strip bin/TrafficPlugin.ld.so &&\
+ strip bin/PCREPlugin.ld.so &&\
+ strip bin/TransparentPlugin.ld.so &&\
+ strip bin/SSLPlugin.ld.so
+RUN tree 3proxy/bin
+RUN ls -l /lib/x86_64-linux-gnu/libdl.so.*
+
+# FROM busybox:glibc
+FROM ubuntu:20.04
+# COPY --from=buildenv /lib/x86_64-linux-gnu/libdl.so.* /lib/ # COPY failed: no source files were specified probably because of the symlink 
+#                                                               /lib/x86_64-linux-gnu/libdl.so.2 -> libdl-2.31.so
+COPY --from=buildenv /lib/x86_64-linux-gnu/libdl-2.31.so /lib/
+RUN ln -s /lib/libdl-2.31.so /lib/libdl.so.2
+
+COPY --from=buildenv 3proxy/bin/* /bin/
+# COPY --from=buildenv 3proxy/bin/3proxy /bin/
+COPY --from=buildenv 3proxy/bin/*.ld.so /usr/local/3proxy/libexec/
+RUN mkdir /usr/local/3proxy/logs &&\
+ mkdir /usr/local/3proxy/conf &&\
+ chown -R 65535:65535 /usr/local/3proxy &&\
+ chmod -R 550  /usr/local/3proxy &&\
+ chmod 750  /usr/local/3proxy/logs &&\
+ chmod -R 555 /usr/local/3proxy/libexec &&\
+ chown -R root /usr/local/3proxy/libexec &&\
+ mkdir /etc/3proxy/ &&\
+ echo chroot /usr/local/3proxy 65535 65535 >/etc/3proxy/3proxy.cfg &&\
+ echo include /conf/3proxy.cfg >>/etc/3proxy/3proxy.cfg &&\
+ chmod 440  /etc/3proxy/3proxy.cfg
+
+
+
+CMD ["/bin/3proxy", "/etc/3proxy/3proxy.cfg"]

tests/transparent-setup/3proxy-HTTP.cfg

@@ -1,3 +1,7 @@
+# this config forwards (transparently) redirected (to port 8011) plaintext HTTP requests via the parent plaintext HTTP upstream proxy to the target host (as specified in the http request itself)
+# For transparently redirected requests it transforms the target after the GET into absolute form before forwarding it to the parent proxy
+# the port 8011 can be also used as http proxy without transparent redirection than it adds proxy auth credentials to the request as specied in the parent config line
+
 plugin /usr/local/3proxy/libexec/TransparentPlugin.ld.so transparent_plugin
 log
 logformat "L[%Y-%m-%dT%H:%M:%S.%.]_HTTP/PLAIN-proxy    || %U@%N:%p src: %C:%c dst: %R:%r orig_dst: %Q:%q ext_iface: %e hops: %h host: %n bytes-I/0: %I/%O byte/sec-I/O: %B/%b duration: %D text: %T error: %E"

tests/transparent-setup/3proxy-TLS.cfg

@@ -0,0 +1,20 @@
+# this config forwards transparently redirected (to port 8012) TLS connections via parent upstream HTTPS-CONNECT proxy based on SNI in TLS client hello
+plugin /usr/local/3proxy/libexec/TransparentPlugin.ld.so transparent_plugin
+log
+logformat "L[%Y-%m-%dT%H:%M:%S.%.]_TLS-proxy    || %U@%N:%p src: %C:%c dst: %R:%r orig_dst: %Q:%q ext_iface: %e hops: %h host: %n bytes-I/0: %I/%O byte/sec-I/O: %B/%b duration: %D text: %T error: %E"
+auth iponly
+#fakeresolve
+
+allow *
+parent 1000 connect+ tools.dbpedia.org 8894 user1 mypassword
+
+transparent
+tlspr -p8012 -P443 -c1
+# tls -p8012 -i127.0.0.1 -e127.0.0.1 -b127.0.0.1 # TODO think about inferfaces and whether to not require 
+# tlspr redirects all incoming TLS connection on port 8012 to the parent proxy using default destination port 443 (in theory - see NOTE for weird behavior)
+# using the required (-c1 option) hostname in SNI extension of TLS client hello as target authority for the upstream proxy (CONNECT request)
+# see https://github.com/3proxy/3proxy/commit/013d4bc3339d8e3760855eae448972f3f34cba14 commit message for the -cN options and other pararemeters
+# NOTE: -P443 does not seem to have an effect neither when using transparent plugin (which is good) nor when not using it (then weird target port is used)
+
+
+

tests/transparent-setup/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.1'
 services:
 
   # Example/demo service for which http (port 80) and https (port 443) traffic will be routed transparently through the proxy
@@ -17,7 +17,9 @@ services:
 
   # Time-machine service for handling HTTPS interception based on the ontology_time_machine image
   time-machine:
-    build: ../..
+    build: 
+      context: . # TODO check whethet that is relative to dockerfile or compose file
+      dockerfile: ../..
     command: --httpsInterception all # This enables HTTPS interception for all traffic
     # ports:
     #   - "8184:8899" # Expose the time machine's port if you want to use the time machine outside of the docker network for debugging
@@ -34,8 +36,8 @@ services:
       - NET_ADMIN     # Required to manipulate network interfaces and set iptables rules for (transparent) traffic redirection
       - NET_RAW       # Allows handling raw packets, which is needed for a transparent proxy to identify the original destination of the redirected traffic
     volumes:
-      - ./3proxy-CONNECT.cfg:/etc/3proxy/3proxy-CONNECT.cfg:ro # Mount the 3proxy configuration for upstream proxy CONNECT requests (https)
-      - ./3proxy-HTTP.cfg:/etc/3proxy/3proxy-HTTP.cfg:ro # Mount the 3proxy configuration for upstream proxy CONNECT requests (https)
+      - ./3proxy-CONNECT.cfg:/etc/3proxy/3proxy-CONNECT.cfg:ro # 3proxy config that transforms https requests to upstream HTTP proxy CONNECT requests
+      - ./3proxy-HTTP.cfg:/etc/3proxy/3proxy-HTTP.cfg:ro # 3proxy config that transforms http request to upstream HTTP proxy requests (absolute URIs)
     networks:
       - my_network
     entrypoint: ["sh", "-c", "
@@ -65,24 +67,24 @@ services:
           /usr/bin/3proxy /etc/3proxy/3proxy-CONNECT.cfg"]
 
   # Transparent Proxy service to intercept traffic and retrieve original destinations
-  transparent-proxy-old:
-    image: 3proxy/3proxy
-    # build: ./transparent-proxy # Assuming there’s a Dockerfile in a subdirectory named transparent-proxy
+  3proxy:
+    # image: 3proxy/3proxy # image is too old and does not support tlspr(oxy)
+    build: 
+      context: .
+      dockerfile: ./3proxy-Dockerfile-full
     cap_add:
       - NET_ADMIN     # Required to manipulate network interfaces and set iptables rules for traffic redirection
       - NET_RAW       # Allows handling raw packets, which is essential for a transparent proxy to work properly
     volumes:
-      - ./3proxy-CONNECT.cfg:/etc/3proxy/3proxy.cfg # Mount the 3proxy configuration for SOCKS5
-    networks:
-      - my_network
+        - ./3proxy-TLS.cfg:/etc/3proxy/3proxy-TLS.cfg:ro # 3proxy config that transforms TLS (with SNI!!!) connections to upstream to tunneled HTTP proxy CONNECT requests
+        - ./3proxy-HTTP.cfg:/etc/3proxy/3proxy-HTTP.cfg:ro # 3proxy config that transforms http request to upstream HTTP proxy requests (absolute URIs)
+    # networks:
+    #   - my_network
+    network_mode: "service:transparent-proxy"
     entrypoint: ["sh", "-c", "
       set -x && \
-      echo applying iptable rules && \
-          apt install -y iptables &&
-          iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 8011 && \
-          iptables -t nat -A PREROUTING -p tcp --dport 443 -j REDIRECT --to-port 8012 && \
       echo starting 3proxy && \
-      exec /usr/bin/3proxy /etc/3proxy/3proxy.cfg"]
+      exec /bin/3proxy /etc/3proxy/3proxy-TLS.cfg"]
 
 
     # entrypoint: ["sh", "-c", "