diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index f11d00a169c..1bf68ab37ff 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -570,6 +570,10 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e corehttp.CommandsROOption(*cctx), } + if cfg.Experimental.P2pHttpProxy { + opts = append(opts, corehttp.ProxyOption()) + } + if len(cfg.Gateway.RootRedirect) > 0 { opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect)) } diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go new file mode 100644 index 00000000000..c9d74fa9244 --- /dev/null +++ b/core/corehttp/proxy.go @@ -0,0 +1,79 @@ +package corehttp + +import ( + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + core "github.com/ipfs/go-ipfs/core" + + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + p2phttp "gx/ipfs/QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD/go-libp2p-http" +) + +// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer +func ProxyOption() ServeOption { + return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + mux.HandleFunc("/p2p/", func(w http.ResponseWriter, request *http.Request) { + // parse request + parsedRequest, err := parseRequest(request) + if err != nil { + handleError(w, "failed to parse request", err, 400) + return + } + + request.Host = "" // Let URL's Host take precedence. + request.URL.Path = parsedRequest.httpPath + target, err := url.Parse(fmt.Sprintf("libp2p://%s", parsedRequest.target)) + if err != nil { + handleError(w, "failed to parse url", err, 400) + return + } + + rt := p2phttp.NewTransport(ipfsNode.PeerHost, p2phttp.ProtocolOption(parsedRequest.name)) + proxy := httputil.NewSingleHostReverseProxy(target) + proxy.Transport = rt + proxy.ServeHTTP(w, request) + }) + return mux, nil + } +} + +type proxyRequest struct { + target string + name protocol.ID + httpPath string // path to send to the proxy-host +} + +// from the url path parse the peer-ID, name and http path +// /p2p/$peer_id/http/$http_path +// or +// /p2p/$peer_id/x/$protocol/http/$http_path +func parseRequest(request *http.Request) (*proxyRequest, error) { + path := request.URL.Path + + split := strings.SplitN(path, "/", 5) + if len(split) < 5 { + return nil, fmt.Errorf("Invalid request path '%s'", path) + } + + if split[3] == "http" { + return &proxyRequest{split[2], protocol.ID("/http"), split[4]}, nil + } + + split = strings.SplitN(path, "/", 7) + if split[3] != "x" || split[5] != "http" { + return nil, fmt.Errorf("Invalid request path '%s'", path) + } + + return &proxyRequest{split[2], protocol.ID("/x/" + split[4] + "/http"), split[6]}, nil +} + +func handleError(w http.ResponseWriter, msg string, err error, code int) { + w.WriteHeader(code) + fmt.Fprintf(w, "%s: %s\n", msg, err) + log.Warningf("http proxy error: %s: %s", err) +} diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go new file mode 100644 index 00000000000..786dcf3d975 --- /dev/null +++ b/core/corehttp/proxy_test.go @@ -0,0 +1,56 @@ +package corehttp + +import ( + "net/http" + "strings" + "testing" + + "github.com/ipfs/go-ipfs/thirdparty/assert" + + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" +) + +type TestCase struct { + urlprefix string + target string + name string + path string +} + +var validtestCases = []TestCase{ + {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/http", "path/to/index.txt"}, + {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "path/to/index.txt"}, + {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "http/path/to/index.txt"}, +} + +func TestParseRequest(t *testing.T) { + for _, tc := range validtestCases { + url := tc.urlprefix + "/p2p/" + tc.target + tc.name + "/" + tc.path + req, _ := http.NewRequest("GET", url, strings.NewReader("")) + + parsed, err := parseRequest(req) + if err != nil { + t.Fatal(err) + } + assert.True(parsed.httpPath == tc.path, t, "proxy request path") + assert.True(parsed.name == protocol.ID(tc.name), t, "proxy request name") + assert.True(parsed.target == tc.target, t, "proxy request peer-id") + } +} + +var invalidtestCases = []string{ + "http://localhost:5001/p2p/http/foobar", + "http://localhost:5001/p2p/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/x/custom/foobar", +} + +func TestParseRequestInvalidPath(t *testing.T) { + for _, tc := range invalidtestCases { + url := tc + req, _ := http.NewRequest("GET", url, strings.NewReader("")) + + _, err := parseRequest(req) + if err == nil { + t.Fail() + } + } +} diff --git a/docs/experimental-features.md b/docs/experimental-features.md index c1f4ef21202..f4fb78fe88d 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -21,6 +21,7 @@ the above issue. - [BadgerDB datastore](#badger-datastore) - [Private Networks](#private-networks) - [ipfs p2p](#ipfs-p2p) +- [p2p http proxy](#p2p-http-proxy) - [Circuit Relay](#circuit-relay) - [Plugins](#plugins) - [Directory Sharding / HAMT](#directory-sharding-hamt) @@ -382,6 +383,87 @@ with `ssh [user]@127.0.0.1 -p 2222`. --- +## p2p http proxy + +Allows proxying of HTTP requests over p2p streams. This allows serving any standard http app over p2p streams. + +### State + +Experimental + +### In Version + +master, 0.4.19 + +### How to enable + +The `p2p` command needs to be enabled in config: + +```sh +> ipfs config --json Experimental.Libp2pStreamMounting true +``` + +On the client, the p2p http proxy needs to be enabled in the config: + +```sh +> ipfs config --json Experimental.P2pHttpProxy true +``` + +### How to use + +**Netcat example:** + +First, pick a protocol name for your application. Think of the protocol name as +a port number, just significantly more user-friendly. In this example, we're +going to use `/http`. + +***Setup:*** + +1. A "server" node with peer ID `$SERVER_ID` +2. A "client" node. + +***On the "server" node:*** + +First, start your application and have it listen for TCP connections on +port `$APP_PORT`. + +Then, configure the p2p listener by running: + +```sh +> ipfs p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$APP_PORT +``` + +This will configure IPFS to forward all incoming `/http` streams to +`127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream. + +***On the "client" node:*** + +Next, have your application make a http request to `127.0.0.1:8080/p2p/$SERVER_ID/http/$FORWARDED_PATH`. This +connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on +the remote machine (which needs to be a http server!) with path `$FORWARDED_PATH`. You can test it with netcat: + +***On "server" node:*** +```sh +> echo -e "HTTP/1.1 200\nContent-length: 11\n\nIPFS rocks!" | nc -l -p $APP_PORT +``` + +***On "client" node:*** +```sh +> curl http://localhost:8080/p2p/$SERVER_ID/http/ +``` + +You should now see the resulting http response: IPFS rocks! + +### Custom protocol names +We also support use of protocol names of the form /x/$NAME/http where $NAME doesn't contain any "/"'s + +### Road to being a real feature +- [ ] Needs p2p streams to graduate from experiments +- [ ] Needs more people to use and report on how well it works / fits use cases +- [ ] More documentation + +--- + ## Circuit Relay Allows peers to connect through an intermediate relay node when there diff --git a/package.json b/package.json index 643b90a4533..f4e722539f8 100644 --- a/package.json +++ b/package.json @@ -592,6 +592,12 @@ "hash": "QmTqLBwme9BusYWdACqL62NFb8WV2Q72gXLsQVfC7vmCr4", "name": "iptb-plugins", "version": "1.0.5" + }, + { + "author": "hsanjuan", + "hash": "QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD", + "name": "go-libp2p-http", + "version": "1.1.8" } ], "gxVersion": "0.10.0", diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh new file mode 100755 index 00000000000..15de5eac1fe --- /dev/null +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash + +test_description="Test http proxy over p2p" + +. lib/test-lib.sh +WEB_SERVE_PORT=5099 +IPFS_GATEWAY_PORT=5199 +SENDER_GATEWAY="http://127.0.0.1:$IPFS_GATEWAY_PORT" + +function show_logs() { + + echo "*****************" + echo " RECEIVER LOG " + echo "*****************" + iptb logs 1 + echo "*****************" + echo " SENDER LOG " + echo "*****************" + iptb logs 0 + echo "*****************" + echo "REMOTE_SERVER LOG" + echo $REMOTE_SERVER_LOG + echo "*****************" + cat $REMOTE_SERVER_LOG +} + +function start_http_server() { + REMOTE_SERVER_LOG="server.log" + rm -f $REMOTE_SERVER_LOG server_stdin + + mkfifo server_stdin + nc -k -l 127.0.0.1 $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG < server_stdin & + REMOTE_SERVER_PID=$! + exec 7>server_stdin + rm server_stdin + + while ! nc -z 127.0.0.1 $WEB_SERVE_PORT; do + go-sleep 100ms + done +} + +function teardown_remote_server() { + exec 7<&- + kill $REMOTE_SERVER_PID > /dev/null 2>&1 + wait $REMOTE_SERVER_PID || true +} + +function serve_content() { + local body=$1 + local status_code=${2:-"200 OK"} + local length=$((1 + ${#body})) + echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" >&7 +} + +function curl_check_response_code() { + local expected_status_code=$1 + local path_stub=${2:-p2p/$RECEIVER_ID/http/index.txt} + local status_code=$(curl -s --write-out %{http_code} --output /dev/null $SENDER_GATEWAY/$path_stub) + + if [[ "$status_code" -ne "$expected_status_code" ]]; + then + echo "Found status-code "$status_code", expected "$expected_status_code + return 1 + fi + + return 0 +} + +function curl_send_proxy_request_and_check_response() { + local expected_status_code=$1 + local expected_content=$2 + + # + # make a request to SENDER_IPFS via the proxy endpoint + # + CONTENT_PATH="retrieved-file" + STATUS_CODE="$(curl -s -o $CONTENT_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" + + # + # check status code + # + if [[ "$STATUS_CODE" -ne "$expected_status_code" ]]; + then + echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code + return 1 + fi + + # + # check content + # + RESPONSE_CONTENT="$(tail -n 1 $CONTENT_PATH)" + if [[ "$RESPONSE_CONTENT" == "$expected_content" ]]; + then + return 0 + else + echo -e "Found response content:\n'"$RESPONSE_CONTENT"'\nthat differs from expected content:\n'"$expected_content"'" + return 1 + fi +} + +function curl_send_multipart_form_request() { + local expected_status_code=$1 + local FILE_PATH="uploaded-file" + FILE_CONTENT="curl will send a multipart-form POST request when sending a file which is handy" + echo $FILE_CONTENT > $FILE_PATH + # + # send multipart form request + # + STATUS_CODE="$(curl -o /dev/null -s -F file=@$FILE_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" + # + # check status code + # + if [[ "$STATUS_CODE" -ne "$expected_status_code" ]]; + then + echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code + return 1 + fi + # + # check request method + # + if ! grep "POST /index.txt" $REMOTE_SERVER_LOG > /dev/null; + then + echo "Remote server request method/resource path was incorrect" + show_logs + return 1 + fi + # + # check request is multipart-form + # + if ! grep "Content-Type: multipart/form-data;" $REMOTE_SERVER_LOG > /dev/null; + then + echo "Request content-type was not multipart/form-data" + show_logs + return 1 + fi + return 0 +} + +test_expect_success 'configure nodes' ' + iptb testbed create -type localipfs -count 2 -force -init && + ipfsi 0 config --json Experimental.Libp2pStreamMounting true && + ipfsi 1 config --json Experimental.Libp2pStreamMounting true && + ipfsi 0 config --json Experimental.P2pHttpProxy true + ipfsi 0 config --json Addresses.Gateway "[\"/ip4/127.0.0.1/tcp/$IPFS_GATEWAY_PORT\"]" +' + +test_expect_success 'start and connect nodes' ' + iptb start -wait && iptb connect 0 1 +' + +test_expect_success 'setup p2p listener on the receiver' ' + ipfsi 1 p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT && + ipfsi 1 p2p listen /x/custom/http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT +' + +test_expect_success 'setup environment' ' + RECEIVER_ID="$(iptb attr get 1 id)" +' + +test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' + curl_send_proxy_request_and_check_response 502 "" +' + +test_expect_success 'start http server' ' + start_http_server +' + +test_expect_success 'handle proxy http request propogates error response from remote' ' + serve_content "SORRY GUYS, I LOST IT" "404 Not Found" && + curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" +' + +test_expect_success 'handle proxy http request ' ' + serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && + curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" +' + +test_expect_success 'handle proxy http request invalid request' ' + curl_check_response_code 400 p2p/DERPDERPDERP +' + +test_expect_success 'handle proxy http request unknown proxy peer ' ' + curl_check_response_code 502 p2p/unknown_peer/http/index.txt +' + +test_expect_success 'handle proxy http request to custom protocol' ' + serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && + curl_check_response_code 200 p2p/$RECEIVER_ID/x/custom/http/index.txt +' + +test_expect_success 'handle proxy http request to missing protocol' ' + serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && + curl_check_response_code 502 p2p/$RECEIVER_ID/x/missing/http/index.txt +' + +test_expect_success 'handle proxy http request missing the /http' ' + curl_check_response_code 400 p2p/$RECEIVER_ID/x/custom/index.txt +' + +test_expect_success 'handle multipart/form-data http request' ' + serve_content "OK" && + curl_send_multipart_form_request 200 +' + +test_expect_success 'stop http server' ' + teardown_remote_server +' + +test_expect_success 'stop nodes' ' + iptb stop +' + +test_done