feat(lib/runtime): Implement `ext_offchain_http_request_add_header_ve…

…rsion_1` host function (ChainSafe#1994)

* feat: implement offchain http host functions

* chore: decoding Result<i16, ()>

* chore: adjust result encoding/decoding

* chore: add export comment on Get

* chore: change to map and update test wasm

* chore: use request id buffer

* chore: change to NewHTTPSet

* chore: add export comment

* chore: use pkg/scale to encode Result to wasm memory

* chore: update naming and fix lint warns

* chore: use buffer.put when remove http request

* chore: add more comments

* chore: add unit tests

* chore: fix misspelling

* chore: fix scale marshal to encode Result instead of Option<Result>

* chore: ignore uneeded error

* chore: fix unused params

* chore: cannot remove unused params

* chore: ignore deepsource errors

* chore: add parallel to wasmer tests

* chore: implementing offchain http request add header

* chore: remove dereferencing

* chore: fix param compatibility

* chore: embed mutex iunto httpset struct

* chore: fix request field name

* chore: update the hoost polkadot test runtime location

* chore: use an updated host runtime test

* chore: fix lint warns

* chore: rename OffchainRequest to Request

* chore: update host commit hash

* chore: update log

* chore: address comments

* chore: adjust the error flow

* chore: fix result return

* chore: update the host runtime link

* chore: use request context to store bool values

* chore: fix the lint issues

README.md

@@ -58,6 +58,8 @@ build gossamer command:
 make gossamer
 ```
 
+
+
 ### Run Development Node
 
 To initialise a development node:

lib/runtime/constants.go

@@ -32,10 +32,9 @@ const (
 	POLKADOT_RUNTIME_URL = "https://github.com/noot/polkadot/blob/noot/v0.8.25/polkadot_runtime.wasm?raw=true"
 
 	// v0.9 test API wasm
-	HOST_API_TEST_RUNTIME    = "hostapi_runtime"
-	HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm"
-	// HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/b94d8c58ad6ea8bf827b0cae1645a999719c2bc7/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true"
-	HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/omar/offchain/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true"
+	HOST_API_TEST_RUNTIME     = "hostapi_runtime"
+	HOST_API_TEST_RUNTIME_FP  = "hostapi_runtime.compact.wasm"
+	HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/4d190603d21d4431888bcb1ec546c4dc03b7bf93/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true"
 
 	// v0.8 substrate runtime with modified name and babe C=(1, 1)
 	DEV_RUNTIME     = "dev_runtime"

lib/runtime/offchain/httpset.go

@@ -4,17 +4,29 @@
 package offchain
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 )
 
+type contextKey string
+
+const (
+	waitingKey contextKey = "waiting"
+	invalidKey contextKey = "invalid"
+)
+
 const maxConcurrentRequests = 1000
 
 var (
 	errIntBufferEmpty        = errors.New("int buffer exhausted")
 	errIntBufferFull         = errors.New("int buffer is full")
 	errRequestIDNotAvailable = errors.New("request id not available")
+	errRequestInvalid        = errors.New("request is invalid")
+	errInvalidHeaderKey      = errors.New("invalid header key")
 )
 
 // requestIDBuffer created to control the amount of available non-duplicated ids
@@ -48,10 +60,32 @@ func (b requestIDBuffer) put(i int16) error {
 	}
 }
 
+// Request holds the request object and update the invalid and waiting status whenever
+// the request starts or is waiting to be read
+type Request struct {
+	Request *http.Request
+}
+
+// AddHeader adds a new HTTP header into request property, only if request is valid
+func (r *Request) AddHeader(name, value string) error {
+	invalid, ok := r.Request.Context().Value(invalidKey).(bool)
+	if ok && invalid {
+		return errRequestInvalid
+	}
+
+	name = strings.TrimSpace(name)
+	if len(name) == 0 {
+		return fmt.Errorf("%w: empty header key", errInvalidHeaderKey)
+	}
+
+	r.Request.Header.Add(name, value)
+	return nil
+}
+
 // HTTPSet holds a pool of concurrent http request calls
 type HTTPSet struct {
 	*sync.Mutex
-	reqs   map[int16]*http.Request
+	reqs   map[int16]*Request
 	idBuff requestIDBuffer
 }
 
@@ -60,7 +94,7 @@ type HTTPSet struct {
 func NewHTTPSet() *HTTPSet {
 	return &HTTPSet{
 		new(sync.Mutex),
-		make(map[int16]*http.Request),
+		make(map[int16]*Request),
 		newIntBuffer(maxConcurrentRequests),
 	}
 }
@@ -81,11 +115,21 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) {
 	}
 
 	req, err := http.NewRequest(method, uri, nil)
+	req.Header = make(http.Header)
+
+	ctx := context.WithValue(req.Context(), waitingKey, false)
+	ctx = context.WithValue(ctx, invalidKey, false)
+
+	req = req.WithContext(ctx)
+
 	if err != nil {
 		return 0, err
 	}
 
-	p.reqs[id] = req
+	p.reqs[id] = &Request{
+		Request: req,
+	}
+
 	return id, nil
 }
 
@@ -100,7 +144,7 @@ func (p *HTTPSet) Remove(id int16) error {
 }
 
 // Get returns a request or nil if request not found
-func (p *HTTPSet) Get(id int16) *http.Request {
+func (p *HTTPSet) Get(id int16) *Request {
 	p.Lock()
 	defer p.Unlock()
 

lib/runtime/offchain/httpset_test.go

@@ -4,6 +4,8 @@
 package offchain
 
 import (
+	"context"
+	"fmt"
 	"net/http"
 	"testing"
 
@@ -28,7 +30,7 @@ func TestHTTPSet_StartRequest_NotAvailableID(t *testing.T) {
 	t.Parallel()
 
 	set := NewHTTPSet()
-	set.reqs[1] = &http.Request{}
+	set.reqs[1] = &Request{}
 
 	_, err := set.StartRequest(http.MethodGet, defaultTestURI)
 	require.ErrorIs(t, errRequestIDNotAvailable, err)
@@ -45,6 +47,57 @@ func TestHTTPSetGet(t *testing.T) {
 	req := set.Get(id)
 	require.NotNil(t, req)
 
-	require.Equal(t, http.MethodGet, req.Method)
-	require.Equal(t, defaultTestURI, req.URL.String())
+	require.Equal(t, http.MethodGet, req.Request.Method)
+	require.Equal(t, defaultTestURI, req.Request.URL.String())
+}
+
+func TestOffchainRequest_AddHeader(t *testing.T) {
+	t.Parallel()
+
+	invalidCtx := context.WithValue(context.Background(), invalidKey, true)
+	invalidReq, err := http.NewRequestWithContext(invalidCtx, http.MethodGet, "http://test.com", nil)
+	require.NoError(t, err)
+
+	cases := map[string]struct {
+		offReq           Request
+		err              error
+		headerK, headerV string
+	}{
+		"should return invalid request": {
+			offReq: Request{invalidReq},
+			err:    errRequestInvalid,
+		},
+		"should add header": {
+			offReq:  Request{Request: &http.Request{Header: make(http.Header)}},
+			headerK: "key",
+			headerV: "value",
+		},
+		"should return invalid empty header": {
+			offReq:  Request{Request: &http.Request{Header: make(http.Header)}},
+			headerK: "",
+			headerV: "value",
+			err:     fmt.Errorf("%w: %s", errInvalidHeaderKey, "empty header key"),
+		},
+	}
+
+	for name, tc := range cases {
+		tc := tc
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			err := tc.offReq.AddHeader(tc.headerK, tc.headerV)
+
+			if tc.err != nil {
+				require.Error(t, err)
+				require.Equal(t, tc.err.Error(), err.Error())
+				return
+			}
+
+			require.NoError(t, err)
+
+			got := tc.offReq.Request.Header.Get(tc.headerK)
+			require.Equal(t, tc.headerV, got)
+		})
+	}
 }

lib/runtime/wasmer/imports.go

@@ -77,6 +77,7 @@ package wasmer
 // extern int64_t ext_offchain_timestamp_version_1(void *context);
 // extern void ext_offchain_sleep_until_version_1(void *context, int64_t a);
 // extern int64_t ext_offchain_http_request_start_version_1(void *context, int64_t a, int64_t b, int64_t c);
+// extern int64_t ext_offchain_http_request_add_header_version_1(void *context, int32_t a, int64_t k, int64_t v);
 //
 // extern void ext_storage_append_version_1(void *context, int64_t a, int64_t b);
 // extern int64_t ext_storage_changes_root_version_1(void *context, int64_t a);
@@ -1722,24 +1723,80 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa
 	logger.Debug("executing...")
 
 	instanceContext := wasm.IntoInstanceContext(context)
+	runtimeCtx := instanceContext.Data().(*runtime.Context)
 
 	httpMethod := asMemorySlice(instanceContext, methodSpan)
 	uri := asMemorySlice(instanceContext, uriSpan)
 
 	result := scale.NewResult(int16(0), nil)
 
-	runtimeCtx := instanceContext.Data().(*runtime.Context)
 	reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri))
-
 	if err != nil {
+		// StartRequest error already was logged
 		logger.Errorf("failed to start request: %s", err)
-		_ = result.Set(scale.Err, nil)
+		err = result.Set(scale.Err, nil)
 	} else {
-		_ = result.Set(scale.OK, reqID)
+		err = result.Set(scale.OK, reqID)
+	}
+
+	// note: just check if an error occurs while setting the result data
+	if err != nil {
+		logger.Errorf("failed to set the result data: %s", err)
+		return C.int64_t(0)
+	}
+
+	enc, err := scale.Marshal(result)
+	if err != nil {
+		logger.Errorf("failed to scale marshal the result: %s", err)
+		return C.int64_t(0)
 	}
 
-	enc, _ := scale.Marshal(result)
-	ptr, _ := toWasmMemory(instanceContext, enc)
+	ptr, err := toWasmMemory(instanceContext, enc)
+	if err != nil {
+		logger.Errorf("failed to allocate result on memory: %s", err)
+		return C.int64_t(0)
+	}
+
+	return C.int64_t(ptr)
+}
+
+//export ext_offchain_http_request_add_header_version_1
+func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqID C.int32_t, nameSpan, valueSpan C.int64_t) C.int64_t {
+	logger.Debug("executing...")
+	instanceContext := wasm.IntoInstanceContext(context)
+
+	name := asMemorySlice(instanceContext, nameSpan)
+	value := asMemorySlice(instanceContext, valueSpan)
+
+	runtimeCtx := instanceContext.Data().(*runtime.Context)
+	offchainReq := runtimeCtx.OffchainHTTPSet.Get(int16(reqID))
+
+	result := scale.NewResult(nil, nil)
+	resultMode := scale.OK
+
+	err := offchainReq.AddHeader(string(name), string(value))
+	if err != nil {
+		logger.Errorf("failed to add request header: %s", err)
+		resultMode = scale.Err
+	}
+
+	err = result.Set(resultMode, nil)
+	if err != nil {
+		logger.Errorf("failed to set the result data: %s", err)
+		return C.int64_t(0)
+	}
+
+	enc, err := scale.Marshal(result)
+	if err != nil {
+		logger.Errorf("failed to scale marshal the result: %s", err)
+		return C.int64_t(0)
+	}
+
+	ptr, err := toWasmMemory(instanceContext, enc)
+	if err != nil {
+		logger.Errorf("failed to allocate result on memory: %s", err)
+		return C.int64_t(0)
+	}
 
 	return C.int64_t(ptr)
 }
@@ -2416,6 +2473,10 @@ func ImportsNodeRuntime() (*wasm.Imports, error) { //nolint
 	if err != nil {
 		return nil, err
 	}
+	_, err = imports.Append("ext_offchain_http_request_add_header_version_1", ext_offchain_http_request_add_header_version_1, C.ext_offchain_http_request_add_header_version_1)
+	if err != nil {
+		return nil, err
+	}
 	_, err = imports.Append("ext_sandbox_instance_teardown_version_1", ext_sandbox_instance_teardown_version_1, C.ext_sandbox_instance_teardown_version_1)
 	if err != nil {
 		return nil, err

lib/runtime/wasmer/imports_test.go

@@ -6,6 +6,7 @@ package wasmer
 import (
 	"bytes"
 	"encoding/binary"
+	"net/http"
 	"os"
 	"sort"
 	"testing"
@@ -312,6 +313,71 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) {
 	require.Equal(t, int16(3), requestNumber)
 }
 
+func Test_ext_offchain_http_request_add_header(t *testing.T) {
+	t.Parallel()
+
+	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)
+
+	cases := map[string]struct {
+		key, value  string
+		expectedErr bool
+	}{
+		"should add headers without problems": {
+			key:         "SOME_HEADER_KEY",
+			value:       "SOME_HEADER_VALUE",
+			expectedErr: false,
+		},
+
+		"should return a result error": {
+			key:         "",
+			value:       "",
+			expectedErr: true,
+		},
+	}
+
+	for tname, tcase := range cases {
+		t.Run(tname, func(t *testing.T) {
+			t.Parallel()
+
+			reqID, err := inst.ctx.OffchainHTTPSet.StartRequest(http.MethodGet, "http://uri.example")
+			require.NoError(t, err)
+
+			encID, err := scale.Marshal(uint32(reqID))
+			require.NoError(t, err)
+
+			encHeaderKey, err := scale.Marshal(tcase.key)
+			require.NoError(t, err)
+
+			encHeaderValue, err := scale.Marshal(tcase.value)
+			require.NoError(t, err)
+
+			params := append([]byte{}, encID...)
+			params = append(params, encHeaderKey...)
+			params = append(params, encHeaderValue...)
+
+			ret, err := inst.Exec("rtm_ext_offchain_http_request_add_header_version_1", params)
+			require.NoError(t, err)
+
+			gotResult := scale.NewResult(nil, nil)
+			err = scale.Unmarshal(ret, &gotResult)
+			require.NoError(t, err)
+
+			ok, err := gotResult.Unwrap()
+			if tcase.expectedErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+
+			offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID)
+			gotValue := offchainReq.Request.Header.Get(tcase.key)
+			require.Equal(t, tcase.value, gotValue)
+
+			require.Nil(t, ok)
+		})
+	}
+}
+
 func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) {
 	t.Parallel()
 	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)