feat(lib/runtime): Implement `ext_offchain_http_request_start_version…

…_1` host function (ChainSafe#1947)

* feat: implement offchain http host functions

* chore: decoding Result<i16, ()>

* chore: adjust result encoding/decoding

* chore: add export comment on Get

* chore: change to map and update test wasm

* chore: use request id buffer

* chore: change to NewHTTPSet

* chore: add export comment

* chore: use pkg/scale to encode Result to wasm memory

* chore: update naming and fix lint warns

* chore: use buffer.put when remove http request

* chore: add more comments

* chore: add unit tests

* chore: fix misspelling

* chore: fix scale marshal to encode Result instead of Option<Result>

* chore: ignore uneeded error

* chore: fix unused params

* chore: cannot remove unused params

* chore: ignore deepsource errors

* chore: add parallel to wasmer tests

* chore: remove dereferencing

* chore: fix param compatibility

* chore: embed mutex iunto httpset struct

* chore: update the hoost polkadot test runtime location

* chore: fix request not available test

lib/runtime/constants.go

@@ -47,7 +47,7 @@ const (
 	// v0.9 test API wasm
 	HOST_API_TEST_RUNTIME     = "hostapi_runtime"
 	HOST_API_TEST_RUNTIME_FP  = "hostapi_runtime.compact.wasm"
-	HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/9cc27bf7b7f21c106000103f8f6b6c51f7fb8353/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true"
+	HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/b94d8c58ad6ea8bf827b0cae1645a999719c2bc7/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true"
 
 	// v0.8 substrate runtime with modified name and babe C=(1, 1)
 	DEV_RUNTIME     = "dev_runtime"

lib/runtime/offchain/httpset.go

@@ -0,0 +1,105 @@
+package offchain
+
+import (
+	"errors"
+	"net/http"
+	"sync"
+)
+
+const maxConcurrentRequests = 1000
+
+var (
+	errIntBufferEmpty        = errors.New("int buffer exhausted")
+	errIntBufferFull         = errors.New("int buffer is full")
+	errRequestIDNotAvailable = errors.New("request id not available")
+)
+
+// requestIDBuffer created to control the amount of available non-duplicated ids
+type requestIDBuffer chan int16
+
+// newIntBuffer creates the request id buffer starting from 1 till @buffSize (by default @buffSize is 1000)
+func newIntBuffer(buffSize int16) requestIDBuffer {
+	b := make(chan int16, buffSize)
+	for i := int16(1); i <= buffSize; i++ {
+		b <- i
+	}
+
+	return b
+}
+
+func (b requestIDBuffer) get() (int16, error) {
+	select {
+	case v := <-b:
+		return v, nil
+	default:
+		return 0, errIntBufferEmpty
+	}
+}
+
+func (b requestIDBuffer) put(i int16) error {
+	select {
+	case b <- i:
+		return nil
+	default:
+		return errIntBufferFull
+	}
+}
+
+// HTTPSet holds a pool of concurrent http request calls
+type HTTPSet struct {
+	*sync.Mutex
+	reqs   map[int16]*http.Request
+	idBuff requestIDBuffer
+}
+
+// NewHTTPSet creates a offchain http set that can be used
+// by runtime as HTTP clients, the max concurrent requests is 1000
+func NewHTTPSet() *HTTPSet {
+	return &HTTPSet{
+		new(sync.Mutex),
+		make(map[int16]*http.Request),
+		newIntBuffer(maxConcurrentRequests),
+	}
+}
+
+// StartRequest create a new request using the method and the uri, adds the request into the list
+// and then return the position of the request inside the list
+func (p *HTTPSet) StartRequest(method, uri string) (int16, error) {
+	p.Lock()
+	defer p.Unlock()
+
+	id, err := p.idBuff.get()
+	if err != nil {
+		return 0, err
+	}
+
+	if _, ok := p.reqs[id]; ok {
+		return 0, errRequestIDNotAvailable
+	}
+
+	req, err := http.NewRequest(method, uri, nil)
+	if err != nil {
+		return 0, err
+	}
+
+	p.reqs[id] = req
+	return id, nil
+}
+
+// Remove just remove a expecific request from reqs
+func (p *HTTPSet) Remove(id int16) error {
+	p.Lock()
+	defer p.Unlock()
+
+	delete(p.reqs, id)
+
+	return p.idBuff.put(id)
+}
+
+// Get returns a request or nil if request not found
+func (p *HTTPSet) Get(id int16) *http.Request {
+	p.Lock()
+	defer p.Unlock()
+
+	return p.reqs[id]
+}

lib/runtime/offchain/httpset_test.go

@@ -0,0 +1,47 @@
+package offchain
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+const defaultTestURI = "http://example.url"
+
+func TestHTTPSetLimit(t *testing.T) {
+	t.Parallel()
+
+	set := NewHTTPSet()
+	var err error
+	for i := 0; i < maxConcurrentRequests+1; i++ {
+		_, err = set.StartRequest(http.MethodGet, defaultTestURI)
+	}
+
+	require.ErrorIs(t, errIntBufferEmpty, err)
+}
+
+func TestHTTPSet_StartRequest_NotAvailableID(t *testing.T) {
+	t.Parallel()
+
+	set := NewHTTPSet()
+	set.reqs[1] = &http.Request{}
+
+	_, err := set.StartRequest(http.MethodGet, defaultTestURI)
+	require.ErrorIs(t, errRequestIDNotAvailable, err)
+}
+
+func TestHTTPSetGet(t *testing.T) {
+	t.Parallel()
+
+	set := NewHTTPSet()
+
+	id, err := set.StartRequest(http.MethodGet, defaultTestURI)
+	require.NoError(t, err)
+
+	req := set.Get(id)
+	require.NotNil(t, req)
+
+	require.Equal(t, http.MethodGet, req.Method)
+	require.Equal(t, defaultTestURI, req.URL.String())
+}

lib/runtime/types.go

@@ -19,6 +19,7 @@ package runtime
 import (
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/keystore"
+	"github.com/ChainSafe/gossamer/lib/runtime/offchain"
 	log "github.com/ChainSafe/log15"
 )
 
@@ -72,14 +73,15 @@ type InstanceConfig struct {
 
 // Context is the context for the wasm interpreter's imported functions
 type Context struct {
-	Storage     Storage
-	Allocator   *FreeingBumpHeapAllocator
-	Keystore    *keystore.GlobalKeystore
-	Validator   bool
-	NodeStorage NodeStorage
-	Network     BasicNetwork
-	Transaction TransactionState
-	SigVerifier *SignatureVerifier
+	Storage         Storage
+	Allocator       *FreeingBumpHeapAllocator
+	Keystore        *keystore.GlobalKeystore
+	Validator       bool
+	NodeStorage     NodeStorage
+	Network         BasicNetwork
+	Transaction     TransactionState
+	SigVerifier     *SignatureVerifier
+	OffchainHTTPSet *offchain.HTTPSet
 }
 
 // NewValidateTransactionError returns an error based on a return value from TaggedTransactionQueueValidateTransaction

lib/runtime/wasmer/imports.go

@@ -89,6 +89,7 @@ package wasmer
 // extern int64_t ext_offchain_submit_transaction_version_1(void *context, int64_t a);
 // extern int64_t ext_offchain_timestamp_version_1(void *context);
 // extern void ext_offchain_sleep_until_version_1(void *context, int64_t a);
+// extern int64_t ext_offchain_http_request_start_version_1(void *context, int64_t a, int64_t b, int64_t c);
 //
 // extern void ext_storage_append_version_1(void *context, int64_t a, int64_t b);
 // extern int64_t ext_storage_changes_root_version_1(void *context, int64_t a);
@@ -894,7 +895,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan
 }
 
 //export ext_trie_blake2_256_verify_proof_version_1
-func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a C.int32_t, b, c, d C.int64_t) C.int32_t {
+func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a C.int32_t, b, c, d C.int64_t) C.int32_t { // skipcq: RVV-B0012
 	logger.Debug("[ext_trie_blake2_256_verify_proof_version_1] executing...")
 	logger.Warn("[ext_trie_blake2_256_verify_proof_version_1] unimplemented")
 	return 0
@@ -1675,6 +1676,33 @@ func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) {
 	logger.Warn("unimplemented")
 }
 
+//export ext_offchain_http_request_start_version_1
+func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, metaSpan C.int64_t) C.int64_t { // skipcq: RVV-B0012
+	logger.Debug("executing...")
+
+	instanceContext := wasm.IntoInstanceContext(context)
+
+	httpMethod := asMemorySlice(instanceContext, methodSpan)
+	uri := asMemorySlice(instanceContext, uriSpan)
+
+	result := scale.NewResult(int16(0), nil)
+
+	runtimeCtx := instanceContext.Data().(*runtime.Context)
+	reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri))
+
+	if err != nil {
+		logger.Error("failed to start request", "error", err)
+		_ = result.Set(scale.Err, nil)
+	} else {
+		_ = result.Set(scale.OK, reqID)
+	}
+
+	enc, _ := scale.Marshal(result)
+	ptr, _ := toWasmMemory(instanceContext, enc)
+
+	return C.int64_t(ptr)
+}
+
 func storageAppend(storage runtime.Storage, key, valueToAppend []byte) error {
 	nextLength := big.NewInt(1)
 	var valueRes []byte
@@ -2334,6 +2362,10 @@ func ImportsNodeRuntime() (*wasm.Imports, error) { //nolint
 	if err != nil {
 		return nil, err
 	}
+	_, err = imports.Append("ext_offchain_http_request_start_version_1", ext_offchain_http_request_start_version_1, C.ext_offchain_http_request_start_version_1)
+	if err != nil {
+		return nil, err
+	}
 	_, err = imports.Append("ext_sandbox_instance_teardown_version_1", ext_sandbox_instance_teardown_version_1, C.ext_sandbox_instance_teardown_version_1)
 	if err != nil {
 		return nil, err