From 0a30b3dd9c040b59a4f5539b410367cbafa5a79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Fri, 19 Nov 2021 22:02:24 -0400 Subject: [PATCH] feat(lib/runtime): Implement `ext_offchain_http_request_add_header_version_1` host function (#1994) * feat: implement offchain http host functions * chore: decoding Result * chore: adjust result encoding/decoding * chore: add export comment on Get * chore: change to map and update test wasm * chore: use request id buffer * chore: change to NewHTTPSet * chore: add export comment * chore: use pkg/scale to encode Result to wasm memory * chore: update naming and fix lint warns * chore: use buffer.put when remove http request * chore: add more comments * chore: add unit tests * chore: fix misspelling * chore: fix scale marshal to encode Result instead of Option * chore: ignore uneeded error * chore: fix unused params * chore: cannot remove unused params * chore: ignore deepsource errors * chore: add parallel to wasmer tests * chore: implementing offchain http request add header * chore: remove dereferencing * chore: fix param compatibility * chore: embed mutex iunto httpset struct * chore: fix request field name * chore: update the hoost polkadot test runtime location * chore: use an updated host runtime test * chore: fix lint warns * chore: rename OffchainRequest to Request * chore: update host commit hash * chore: update log * chore: address comments * chore: adjust the error flow * chore: fix result return * chore: update the host runtime link * chore: use request context to store bool values * chore: fix the lint issues --- README.md | 2 + lib/runtime/constants.go | 7 ++- lib/runtime/offchain/httpset.go | 52 ++++++++++++++++++-- lib/runtime/offchain/httpset_test.go | 59 ++++++++++++++++++++-- lib/runtime/wasmer/imports.go | 73 +++++++++++++++++++++++++--- lib/runtime/wasmer/imports_test.go | 66 +++++++++++++++++++++++++ 6 files changed, 242 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 9db92e7010..3cc74233df 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ build gossamer command: make gossamer ``` + + ### Run Development Node To initialise a development node: diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index e1298a5ffc..f2b7d9cb7d 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -32,10 +32,9 @@ const ( POLKADOT_RUNTIME_URL = "https://github.com/noot/polkadot/blob/noot/v0.8.25/polkadot_runtime.wasm?raw=true" // v0.9 test API wasm - HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - // HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/b94d8c58ad6ea8bf827b0cae1645a999719c2bc7/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/omar/offchain/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME = "hostapi_runtime" + HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/4d190603d21d4431888bcb1ec546c4dc03b7bf93/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 371af26e68..eb47059709 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -4,17 +4,29 @@ package offchain import ( + "context" "errors" + "fmt" "net/http" + "strings" "sync" ) +type contextKey string + +const ( + waitingKey contextKey = "waiting" + invalidKey contextKey = "invalid" +) + const maxConcurrentRequests = 1000 var ( errIntBufferEmpty = errors.New("int buffer exhausted") errIntBufferFull = errors.New("int buffer is full") errRequestIDNotAvailable = errors.New("request id not available") + errRequestInvalid = errors.New("request is invalid") + errInvalidHeaderKey = errors.New("invalid header key") ) // requestIDBuffer created to control the amount of available non-duplicated ids @@ -48,10 +60,32 @@ func (b requestIDBuffer) put(i int16) error { } } +// Request holds the request object and update the invalid and waiting status whenever +// the request starts or is waiting to be read +type Request struct { + Request *http.Request +} + +// AddHeader adds a new HTTP header into request property, only if request is valid +func (r *Request) AddHeader(name, value string) error { + invalid, ok := r.Request.Context().Value(invalidKey).(bool) + if ok && invalid { + return errRequestInvalid + } + + name = strings.TrimSpace(name) + if len(name) == 0 { + return fmt.Errorf("%w: empty header key", errInvalidHeaderKey) + } + + r.Request.Header.Add(name, value) + return nil +} + // HTTPSet holds a pool of concurrent http request calls type HTTPSet struct { *sync.Mutex - reqs map[int16]*http.Request + reqs map[int16]*Request idBuff requestIDBuffer } @@ -60,7 +94,7 @@ type HTTPSet struct { func NewHTTPSet() *HTTPSet { return &HTTPSet{ new(sync.Mutex), - make(map[int16]*http.Request), + make(map[int16]*Request), newIntBuffer(maxConcurrentRequests), } } @@ -81,11 +115,21 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { } req, err := http.NewRequest(method, uri, nil) + req.Header = make(http.Header) + + ctx := context.WithValue(req.Context(), waitingKey, false) + ctx = context.WithValue(ctx, invalidKey, false) + + req = req.WithContext(ctx) + if err != nil { return 0, err } - p.reqs[id] = req + p.reqs[id] = &Request{ + Request: req, + } + return id, nil } @@ -100,7 +144,7 @@ func (p *HTTPSet) Remove(id int16) error { } // Get returns a request or nil if request not found -func (p *HTTPSet) Get(id int16) *http.Request { +func (p *HTTPSet) Get(id int16) *Request { p.Lock() defer p.Unlock() diff --git a/lib/runtime/offchain/httpset_test.go b/lib/runtime/offchain/httpset_test.go index 07d1a08773..d9b40c8bf2 100644 --- a/lib/runtime/offchain/httpset_test.go +++ b/lib/runtime/offchain/httpset_test.go @@ -4,6 +4,8 @@ package offchain import ( + "context" + "fmt" "net/http" "testing" @@ -28,7 +30,7 @@ func TestHTTPSet_StartRequest_NotAvailableID(t *testing.T) { t.Parallel() set := NewHTTPSet() - set.reqs[1] = &http.Request{} + set.reqs[1] = &Request{} _, err := set.StartRequest(http.MethodGet, defaultTestURI) require.ErrorIs(t, errRequestIDNotAvailable, err) @@ -45,6 +47,57 @@ func TestHTTPSetGet(t *testing.T) { req := set.Get(id) require.NotNil(t, req) - require.Equal(t, http.MethodGet, req.Method) - require.Equal(t, defaultTestURI, req.URL.String()) + require.Equal(t, http.MethodGet, req.Request.Method) + require.Equal(t, defaultTestURI, req.Request.URL.String()) +} + +func TestOffchainRequest_AddHeader(t *testing.T) { + t.Parallel() + + invalidCtx := context.WithValue(context.Background(), invalidKey, true) + invalidReq, err := http.NewRequestWithContext(invalidCtx, http.MethodGet, "http://test.com", nil) + require.NoError(t, err) + + cases := map[string]struct { + offReq Request + err error + headerK, headerV string + }{ + "should return invalid request": { + offReq: Request{invalidReq}, + err: errRequestInvalid, + }, + "should add header": { + offReq: Request{Request: &http.Request{Header: make(http.Header)}}, + headerK: "key", + headerV: "value", + }, + "should return invalid empty header": { + offReq: Request{Request: &http.Request{Header: make(http.Header)}}, + headerK: "", + headerV: "value", + err: fmt.Errorf("%w: %s", errInvalidHeaderKey, "empty header key"), + }, + } + + for name, tc := range cases { + tc := tc + + t.Run(name, func(t *testing.T) { + t.Parallel() + + err := tc.offReq.AddHeader(tc.headerK, tc.headerV) + + if tc.err != nil { + require.Error(t, err) + require.Equal(t, tc.err.Error(), err.Error()) + return + } + + require.NoError(t, err) + + got := tc.offReq.Request.Header.Get(tc.headerK) + require.Equal(t, tc.headerV, got) + }) + } } diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index b3f292d652..4588fe84c9 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -77,6 +77,7 @@ package wasmer // extern int64_t ext_offchain_timestamp_version_1(void *context); // extern void ext_offchain_sleep_until_version_1(void *context, int64_t a); // extern int64_t ext_offchain_http_request_start_version_1(void *context, int64_t a, int64_t b, int64_t c); +// extern int64_t ext_offchain_http_request_add_header_version_1(void *context, int32_t a, int64_t k, int64_t v); // // extern void ext_storage_append_version_1(void *context, int64_t a, int64_t b); // extern int64_t ext_storage_changes_root_version_1(void *context, int64_t a); @@ -1722,24 +1723,80 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa logger.Debug("executing...") instanceContext := wasm.IntoInstanceContext(context) + runtimeCtx := instanceContext.Data().(*runtime.Context) httpMethod := asMemorySlice(instanceContext, methodSpan) uri := asMemorySlice(instanceContext, uriSpan) result := scale.NewResult(int16(0), nil) - runtimeCtx := instanceContext.Data().(*runtime.Context) reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) - if err != nil { + // StartRequest error already was logged logger.Errorf("failed to start request: %s", err) - _ = result.Set(scale.Err, nil) + err = result.Set(scale.Err, nil) } else { - _ = result.Set(scale.OK, reqID) + err = result.Set(scale.OK, reqID) + } + + // note: just check if an error occurs while setting the result data + if err != nil { + logger.Errorf("failed to set the result data: %s", err) + return C.int64_t(0) + } + + enc, err := scale.Marshal(result) + if err != nil { + logger.Errorf("failed to scale marshal the result: %s", err) + return C.int64_t(0) } - enc, _ := scale.Marshal(result) - ptr, _ := toWasmMemory(instanceContext, enc) + ptr, err := toWasmMemory(instanceContext, enc) + if err != nil { + logger.Errorf("failed to allocate result on memory: %s", err) + return C.int64_t(0) + } + + return C.int64_t(ptr) +} + +//export ext_offchain_http_request_add_header_version_1 +func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqID C.int32_t, nameSpan, valueSpan C.int64_t) C.int64_t { + logger.Debug("executing...") + instanceContext := wasm.IntoInstanceContext(context) + + name := asMemorySlice(instanceContext, nameSpan) + value := asMemorySlice(instanceContext, valueSpan) + + runtimeCtx := instanceContext.Data().(*runtime.Context) + offchainReq := runtimeCtx.OffchainHTTPSet.Get(int16(reqID)) + + result := scale.NewResult(nil, nil) + resultMode := scale.OK + + err := offchainReq.AddHeader(string(name), string(value)) + if err != nil { + logger.Errorf("failed to add request header: %s", err) + resultMode = scale.Err + } + + err = result.Set(resultMode, nil) + if err != nil { + logger.Errorf("failed to set the result data: %s", err) + return C.int64_t(0) + } + + enc, err := scale.Marshal(result) + if err != nil { + logger.Errorf("failed to scale marshal the result: %s", err) + return C.int64_t(0) + } + + ptr, err := toWasmMemory(instanceContext, enc) + if err != nil { + logger.Errorf("failed to allocate result on memory: %s", err) + return C.int64_t(0) + } return C.int64_t(ptr) } @@ -2416,6 +2473,10 @@ func ImportsNodeRuntime() (*wasm.Imports, error) { //nolint if err != nil { return nil, err } + _, err = imports.Append("ext_offchain_http_request_add_header_version_1", ext_offchain_http_request_add_header_version_1, C.ext_offchain_http_request_add_header_version_1) + if err != nil { + return nil, err + } _, err = imports.Append("ext_sandbox_instance_teardown_version_1", ext_sandbox_instance_teardown_version_1, C.ext_sandbox_instance_teardown_version_1) if err != nil { return nil, err diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 0742ba1834..ae773df984 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -6,6 +6,7 @@ package wasmer import ( "bytes" "encoding/binary" + "net/http" "os" "sort" "testing" @@ -312,6 +313,71 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { require.Equal(t, int16(3), requestNumber) } +func Test_ext_offchain_http_request_add_header(t *testing.T) { + t.Parallel() + + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + cases := map[string]struct { + key, value string + expectedErr bool + }{ + "should add headers without problems": { + key: "SOME_HEADER_KEY", + value: "SOME_HEADER_VALUE", + expectedErr: false, + }, + + "should return a result error": { + key: "", + value: "", + expectedErr: true, + }, + } + + for tname, tcase := range cases { + t.Run(tname, func(t *testing.T) { + t.Parallel() + + reqID, err := inst.ctx.OffchainHTTPSet.StartRequest(http.MethodGet, "http://uri.example") + require.NoError(t, err) + + encID, err := scale.Marshal(uint32(reqID)) + require.NoError(t, err) + + encHeaderKey, err := scale.Marshal(tcase.key) + require.NoError(t, err) + + encHeaderValue, err := scale.Marshal(tcase.value) + require.NoError(t, err) + + params := append([]byte{}, encID...) + params = append(params, encHeaderKey...) + params = append(params, encHeaderValue...) + + ret, err := inst.Exec("rtm_ext_offchain_http_request_add_header_version_1", params) + require.NoError(t, err) + + gotResult := scale.NewResult(nil, nil) + err = scale.Unmarshal(ret, &gotResult) + require.NoError(t, err) + + ok, err := gotResult.Unwrap() + if tcase.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID) + gotValue := offchainReq.Request.Header.Get(tcase.key) + require.Equal(t, tcase.value, gotValue) + + require.Nil(t, ok) + }) + } +} + func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)