From cad92086fc12effac12025510178ecd6f8f64f68 Mon Sep 17 00:00:00 2001 From: gyuguen Date: Mon, 13 Mar 2023 11:48:09 +0900 Subject: [PATCH 1/6] feat: add maxConnections and maxRequestBodySize limit setting. --- config/config.go | 16 +++-- server/middleware/limit.go | 31 ++++++++++ server/middleware/limit_test.go | 100 ++++++++++++++++++++++++++++++++ server/server.go | 26 +++++++-- server/service/netutil_test.go | 90 ++++++++++++++++++++++++++++ 5 files changed, 253 insertions(+), 10 deletions(-) create mode 100644 server/middleware/limit.go create mode 100644 server/middleware/limit_test.go create mode 100644 server/service/netutil_test.go diff --git a/config/config.go b/config/config.go index 3b69233..06d2b36 100644 --- a/config/config.go +++ b/config/config.go @@ -48,9 +48,11 @@ type IPFSConfig struct { } type APIConfig struct { - ListenAddr string `mapstructure:"listen-addr"` - WriteTimeout int64 `mapstructure:"write-timeout"` - ReadTimeout int64 `mapstructure:"read-timeout"` + ListenAddr string `mapstructure:"listen-addr"` + WriteTimeout int64 `mapstructure:"write-timeout"` + ReadTimeout int64 `mapstructure:"read-timeout"` + MaxConnections int `mapstructure:"max-connections"` + MaxRequestBodySize int64 `mapstructure:"max-request-body-size"` } func DefaultConfig() *Config { @@ -81,9 +83,11 @@ func DefaultConfig() *Config { IPFSNodeAddr: "127.0.0.1:5001", }, API: APIConfig{ - ListenAddr: "127.0.0.1:8080", - WriteTimeout: 60, - ReadTimeout: 15, + ListenAddr: "127.0.0.1:8080", + WriteTimeout: 60, + ReadTimeout: 15, + MaxConnections: 50, + MaxRequestBodySize: 4 << (10 * 2), // 4MB }, } } diff --git a/server/middleware/limit.go b/server/middleware/limit.go new file mode 100644 index 0000000..69e8728 --- /dev/null +++ b/server/middleware/limit.go @@ -0,0 +1,31 @@ +package middleware + +import ( + "net/http" +) + +type limitMiddleware struct { + maxRequestBodySize int64 +} + +func NewLimitMiddleware(maxRequestBodySize int64) *limitMiddleware { + return &limitMiddleware{ + maxRequestBodySize, + } +} + +// Middleware limits the request body size. +// This is done by first constraining to the ContentLength of the request headder, +// and then reading the actual Body to constraint it. +func (mw *limitMiddleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ContentLength > mw.maxRequestBodySize { + http.Error(w, "request body too large", http.StatusBadRequest) + return + } + r.Body = http.MaxBytesReader(w, r.Body, mw.maxRequestBodySize) + defer r.Body.Close() + + next.ServeHTTP(w, r) + }) +} diff --git a/server/middleware/limit_test.go b/server/middleware/limit_test.go new file mode 100644 index 0000000..c99c9d6 --- /dev/null +++ b/server/middleware/limit_test.go @@ -0,0 +1,100 @@ +package middleware_test + +import ( + "bytes" + "crypto/rand" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/medibloc/panacea-oracle/server/middleware" + "github.com/stretchr/testify/require" +) + +func TestBodySizeSmallerThanLimitSetting(t *testing.T) { + testLimitMiddlewareHTTPRequest( + t, + newRequest(newRandomBody(1023)), + 1024, + http.StatusOK, + "", + ) +} + +func TestBodySizeSameLimitSetting(t *testing.T) { + testLimitMiddlewareHTTPRequest( + t, + newRequest(newRandomBody(1024)), + 1024, + http.StatusOK, + "", + ) +} + +func TestBodySizeLargeThanLimitSetting(t *testing.T) { + testLimitMiddlewareHTTPRequest( + t, + newRequest(newRandomBody(1025)), + 1024, + http.StatusBadRequest, + "request body too large", + ) +} + +func TestDifferentBodySizeAndHeaderContentSize(t *testing.T) { + req := newRequest(newRandomBody(1025)) + req.ContentLength = 1024 + + testLimitMiddlewareHTTPRequest( + t, + req, + 1024, + http.StatusBadRequest, + "request body too large", + ) +} + +func newRandomBody(size int) []byte { + body := make([]byte, size) + rand.Read(body) + + return body +} + +func newRequest(body []byte) *http.Request { + return httptest.NewRequest( + "GET", + "http://test.com", + bytes.NewBuffer(body), + ) +} + +func testLimitMiddlewareHTTPRequest( + t *testing.T, + req *http.Request, + maxRequestBodySize int64, + statusCode int, + bodyMsg string, +) { + w := httptest.NewRecorder() + mw := middleware.NewLimitMiddleware(maxRequestBodySize) + testHandler := mw.Middleware( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + }), + ) + + testHandler.ServeHTTP(w, req) + + resp := w.Result() + require.Equal(t, statusCode, resp.StatusCode) + if bodyMsg != "" { + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Contains(t, string(body), bodyMsg) + } +} diff --git a/server/server.go b/server/server.go index 047268a..90fecfb 100644 --- a/server/server.go +++ b/server/server.go @@ -1,7 +1,10 @@ package server import ( + "fmt" + "net" "net/http" + "net/url" "time" "github.com/gorilla/mux" @@ -11,19 +14,24 @@ import ( "github.com/medibloc/panacea-oracle/server/service/status" "github.com/medibloc/panacea-oracle/service" log "github.com/sirupsen/logrus" + "golang.org/x/net/netutil" ) type Server struct { *http.Server + maxConnections int } func New(svc service.Service) *Server { router := mux.NewRouter() + conf := svc.Config() + limitMiddleware := middleware.NewLimitMiddleware(conf.API.MaxRequestBodySize) jwtAuthMiddleware := middleware.NewJWTAuthMiddleware(svc.QueryClient()) dealRouter := router.PathPrefix("/v0/data-deal").Subrouter() dealRouter.Use( + limitMiddleware.Middleware, jwtAuthMiddleware.Middleware, ) @@ -35,14 +43,24 @@ func New(svc service.Service) *Server { return &Server{ &http.Server{ Handler: router, - Addr: svc.Config().API.ListenAddr, - WriteTimeout: time.Duration(svc.Config().API.WriteTimeout) * time.Second, - ReadTimeout: time.Duration(svc.Config().API.ReadTimeout) * time.Second, + Addr: conf.API.ListenAddr, + WriteTimeout: time.Duration(conf.API.WriteTimeout) * time.Second, + ReadTimeout: time.Duration(conf.API.ReadTimeout) * time.Second, }, + conf.API.MaxConnections, } } func (srv *Server) Run() error { + listenURL, err := url.Parse(srv.Addr) + if err != nil { + return fmt.Errorf("failed to parse api address. address: %s, %w", srv.Addr, err) + } + lis, err := net.Listen(listenURL.Scheme, listenURL.Host) + if err != nil { + return err + } + log.Infof("HTTP server is started: %s", srv.Addr) - return srv.ListenAndServe() + return srv.Serve(netutil.LimitListener(lis, srv.maxConnections)) } diff --git a/server/service/netutil_test.go b/server/service/netutil_test.go new file mode 100644 index 0000000..1195f40 --- /dev/null +++ b/server/service/netutil_test.go @@ -0,0 +1,90 @@ +package service_test + +import ( + "io" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/net/netutil" +) + +func TestNetUtil(t *testing.T) { + + lis := &fakeListener{timeWait: 1} + + limitLis := netutil.LimitListener(lis, 2) + + wg := &sync.WaitGroup{} + start := time.Now() + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + conn, err := limitLis.Accept() + require.NoError(t, err) + defer conn.Close() + }(i) + } + + wg.Wait() + end := time.Now() + + // Send 10 requests, process 2 at a time, and take 1 second per request. + // This request test should take 5 to 6 seconds. + require.True(t, start.Add(time.Second*5).Before(end)) + require.True(t, start.Add(time.Second*6).After(end)) + +} + +type fakeListener struct { + timeWait int64 +} + +// Accept waits for and returns the next connection to the listener. +func (l *fakeListener) Accept() (net.Conn, error) { + time.Sleep(time.Second * time.Duration(l.timeWait)) + + return fakeNetConn{}, nil +} + +// Close closes the listener. +// Any blocked Accept operations will be unblocked and return errors. +func (l *fakeListener) Close() error { + return nil +} + +// Addr returns the listener's network address. +func (l *fakeListener) Addr() net.Addr { + return fakeAddr(1) +} + +type fakeNetConn struct { + io.Reader + io.Writer +} + +func (c fakeNetConn) Close() error { return nil } +func (c fakeNetConn) LocalAddr() net.Addr { return localAddr } +func (c fakeNetConn) RemoteAddr() net.Addr { return remoteAddr } +func (c fakeNetConn) SetDeadline(t time.Time) error { return nil } +func (c fakeNetConn) SetReadDeadline(t time.Time) error { return nil } +func (c fakeNetConn) SetWriteDeadline(t time.Time) error { return nil } + +type fakeAddr int + +var ( + localAddr = fakeAddr(1) + remoteAddr = fakeAddr(2) +) + +func (a fakeAddr) Network() string { + return "net" +} + +func (a fakeAddr) String() string { + return "str" +} From d161470681ecda35371ee15a6991035d426dd7b2 Mon Sep 17 00:00:00 2001 From: gyuguen Date: Mon, 13 Mar 2023 11:52:24 +0900 Subject: [PATCH 2/6] fix --- Makefile | 2 +- server/middleware/limit_test.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 17d0f78..716b341 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ test: $(GO) test -v ./... lint: - GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint + #GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint golangci-lint run --timeout 5m0s --allow-parallel-runners find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" | xargs gofmt -d -s go mod verify diff --git a/server/middleware/limit_test.go b/server/middleware/limit_test.go index c99c9d6..cbac012 100644 --- a/server/middleware/limit_test.go +++ b/server/middleware/limit_test.go @@ -57,7 +57,9 @@ func TestDifferentBodySizeAndHeaderContentSize(t *testing.T) { func newRandomBody(size int) []byte { body := make([]byte, size) - rand.Read(body) + if _, err := rand.Read(body); err != nil { + panic(err) + } return body } From 0b7f6cb75fbf03727ec233fc68d58fb99a9a2cc5 Mon Sep 17 00:00:00 2001 From: gyuguen Date: Mon, 13 Mar 2023 11:52:47 +0900 Subject: [PATCH 3/6] fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 716b341..17d0f78 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ test: $(GO) test -v ./... lint: - #GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint + GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint golangci-lint run --timeout 5m0s --allow-parallel-runners find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" | xargs gofmt -d -s go mod verify From 332a8e173ad7e772d104feaed73a10f64959ecf1 Mon Sep 17 00:00:00 2001 From: gyuguen Date: Mon, 13 Mar 2023 11:56:46 +0900 Subject: [PATCH 4/6] fix --- config/toml.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/toml.go b/config/toml.go index 91eb0c8..9f87ecb 100644 --- a/config/toml.go +++ b/config/toml.go @@ -70,6 +70,8 @@ ipfs-node-addr = "{{ .IPFS.IPFSNodeAddr }}" listen-addr = "{{ .API.ListenAddr }}" write-timeout = "{{ .API.WriteTimeout }}" read-timeout = "{{ .API.ReadTimeout }}" +max-connections = "{{ .API.MaxConnections }}" +max-request-body-size = "{{ .API.MaxRequestBodySize }}" ` var configTemplate *template.Template From dcd725d08a9b913c3dc67d60cfd303783af7f32b Mon Sep 17 00:00:00 2001 From: gyuguen Date: Mon, 13 Mar 2023 12:12:45 +0900 Subject: [PATCH 5/6] fix --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 06d2b36..530db5c 100644 --- a/config/config.go +++ b/config/config.go @@ -83,7 +83,7 @@ func DefaultConfig() *Config { IPFSNodeAddr: "127.0.0.1:5001", }, API: APIConfig{ - ListenAddr: "127.0.0.1:8080", + ListenAddr: "http://127.0.0.1:8080", WriteTimeout: 60, ReadTimeout: 15, MaxConnections: 50, From 769c3ddb3137d279faff7bdbe76567d3eb259a75 Mon Sep 17 00:00:00 2001 From: gyuguen Date: Mon, 13 Mar 2023 12:16:23 +0900 Subject: [PATCH 6/6] fix --- config/config.go | 2 +- server/{service => }/netutil_test.go | 2 +- server/server.go | 10 ++++------ 3 files changed, 6 insertions(+), 8 deletions(-) rename server/{service => }/netutil_test.go (98%) diff --git a/config/config.go b/config/config.go index 530db5c..06d2b36 100644 --- a/config/config.go +++ b/config/config.go @@ -83,7 +83,7 @@ func DefaultConfig() *Config { IPFSNodeAddr: "127.0.0.1:5001", }, API: APIConfig{ - ListenAddr: "http://127.0.0.1:8080", + ListenAddr: "127.0.0.1:8080", WriteTimeout: 60, ReadTimeout: 15, MaxConnections: 50, diff --git a/server/service/netutil_test.go b/server/netutil_test.go similarity index 98% rename from server/service/netutil_test.go rename to server/netutil_test.go index 1195f40..8a21444 100644 --- a/server/service/netutil_test.go +++ b/server/netutil_test.go @@ -1,4 +1,4 @@ -package service_test +package server_test import ( "io" diff --git a/server/server.go b/server/server.go index 90fecfb..0a33f8c 100644 --- a/server/server.go +++ b/server/server.go @@ -1,10 +1,8 @@ package server import ( - "fmt" "net" "net/http" - "net/url" "time" "github.com/gorilla/mux" @@ -52,11 +50,11 @@ func New(svc service.Service) *Server { } func (srv *Server) Run() error { - listenURL, err := url.Parse(srv.Addr) - if err != nil { - return fmt.Errorf("failed to parse api address. address: %s, %w", srv.Addr, err) + addr := srv.Addr + if addr == "" { + addr = ":http" } - lis, err := net.Listen(listenURL.Scheme, listenURL.Host) + lis, err := net.Listen("tcp", addr) if err != nil { return err }