From 7a05ce6818c8261fccea37a8ad966b033006ebfc Mon Sep 17 00:00:00 2001 From: Ivan Danyliuk Date: Wed, 22 Aug 2018 12:09:35 +0300 Subject: [PATCH 01/61] Add drop-in lib package for usage with gomobile --- mobile/response.go | 64 ++++++++ mobile/response_test.go | 27 ++++ mobile/status.go | 338 ++++++++++++++++++++++++++++++++++++++++ mobile/types.go | 77 +++++++++ mobile/utils.go | 16 ++ 5 files changed, 522 insertions(+) create mode 100644 mobile/response.go create mode 100644 mobile/response_test.go create mode 100644 mobile/status.go create mode 100644 mobile/types.go create mode 100644 mobile/utils.go diff --git a/mobile/response.go b/mobile/response.go new file mode 100644 index 00000000000..850124456cb --- /dev/null +++ b/mobile/response.go @@ -0,0 +1,64 @@ +package status + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/status-im/status-go/account" + "github.com/status-im/status-go/transactions" +) + +const ( + codeUnknown int = iota + // special codes + codeFailedParseResponse + codeFailedParseParams + // account related codes + codeErrNoAccountSelected + codeErrInvalidTxSender + codeErrDecrypt +) + +var errToCodeMap = map[error]int{ + account.ErrNoAccountSelected: codeErrNoAccountSelected, + transactions.ErrInvalidTxSender: codeErrInvalidTxSender, + keystore.ErrDecrypt: codeErrDecrypt, +} + +type jsonrpcSuccessfulResponse struct { + Result interface{} `json:"result"` +} + +type jsonrpcErrorResponse struct { + Error jsonError `json:"error"` +} + +type jsonError struct { + Code int `json:"code,omitempty"` + Message string `json:"message"` +} + +func prepareJSONResponse(result interface{}, err error) string { + code := codeUnknown + if c, ok := errToCodeMap[err]; ok { + code = c + } + + return prepareJSONResponseWithCode(result, err, code) +} + +func prepareJSONResponseWithCode(result interface{}, err error, code int) string { + if err != nil { + errResponse := jsonrpcErrorResponse{ + Error: jsonError{Code: code, Message: err.Error()}, + } + response, _ := json.Marshal(&errResponse) + return string(response) + } + + data, err := json.Marshal(jsonrpcSuccessfulResponse{result}) + if err != nil { + return prepareJSONResponseWithCode(nil, err, codeFailedParseResponse) + } + return string(data) +} diff --git a/mobile/response_test.go b/mobile/response_test.go new file mode 100644 index 00000000000..c417d853726 --- /dev/null +++ b/mobile/response_test.go @@ -0,0 +1,27 @@ +package status + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +type nonJSON struct{} + +func (*nonJSON) MarshalJSON() ([]byte, error) { + return nil, errors.New("invalid JSON") +} + +func TestPrepareJSONResponseErrorWithResult(t *testing.T) { + data := prepareJSONResponse("0x123", nil) + require.Equal(t, `{"result":"0x123"}`, data) + + data = prepareJSONResponse(&nonJSON{}, nil) + require.Contains(t, data, `{"error":{"code":1,"message":`) +} + +func TestPrepareJSONResponseErrorWithError(t *testing.T) { + data := prepareJSONResponse("0x123", errors.New("some error")) + require.Contains(t, data, `{"error":{"message":"some error"}}`) +} diff --git a/mobile/status.go b/mobile/status.go new file mode 100644 index 00000000000..209c08bb71d --- /dev/null +++ b/mobile/status.go @@ -0,0 +1,338 @@ +package status + +import ( + "encoding/json" + "fmt" + "os" + "unsafe" + + "github.com/NaySoftware/go-fcm" + "github.com/ethereum/go-ethereum/log" + "github.com/status-im/status-go/api" + "github.com/status-im/status-go/logutils" + "github.com/status-im/status-go/params" + "github.com/status-im/status-go/profiling" + "github.com/status-im/status-go/services/personal" + "github.com/status-im/status-go/signal" + "github.com/status-im/status-go/transactions" + validator "gopkg.in/go-playground/validator.v9" +) + +var statusBackend = api.NewStatusBackend() + +// All general log messages in this package should be routed through this logger. +var logger = log.New("package", "status-go/lib") + +//GenerateConfig for status node +func GenerateConfig(datadir string, networkID int) string { + config, err := params.NewNodeConfig(datadir, "", params.FleetBeta, uint64(networkID)) + if err != nil { + return makeJSONResponse(err) + } + + outBytes, err := json.Marshal(config) + if err != nil { + return makeJSONResponse(err) + } + + return string(outBytes) +} + +//StartNode - start Status node +func StartNode(configJSON string) string { + config, err := params.LoadNodeConfig(configJSON) + if err != nil { + return makeJSONResponse(err) + } + + if err := logutils.OverrideRootLog(true, "INFO", "", true); err != nil { + return makeJSONResponse(err) + } + + api.RunAsync(func() error { return statusBackend.StartNode(config) }) + + return makeJSONResponse(nil) +} + +//StopNode - stop status node +func StopNode() string { + api.RunAsync(statusBackend.StopNode) + return makeJSONResponse(nil) +} + +//ValidateNodeConfig validates config for status node +func ValidateNodeConfig(configJSON string) string { + var resp APIDetailedResponse + + _, err := params.LoadNodeConfig(configJSON) + + // Convert errors to APIDetailedResponse + switch err := err.(type) { + case validator.ValidationErrors: + resp = APIDetailedResponse{ + Message: "validation: validation failed", + FieldErrors: make([]APIFieldError, len(err)), + } + + for i, ve := range err { + resp.FieldErrors[i] = APIFieldError{ + Parameter: ve.Namespace(), + Errors: []APIError{ + { + Message: fmt.Sprintf("field validation failed on the '%s' tag", ve.Tag()), + }, + }, + } + } + case error: + resp = APIDetailedResponse{ + Message: fmt.Sprintf("validation: %s", err.Error()), + } + case nil: + resp = APIDetailedResponse{ + Status: true, + } + } + + respJSON, err := json.Marshal(resp) + if err != nil { + return makeJSONResponse(err) + } + + return string(respJSON) +} + +//ResetChainData remove chain data from data directory +func ResetChainData() string { + api.RunAsync(statusBackend.ResetChainData) + return makeJSONResponse(nil) +} + +//CallRPC calls public APIs via RPC +func CallRPC(inputJSON string) string { + return statusBackend.CallRPC(inputJSON) +} + +//CallPrivateRPC calls both public and private APIs via RPC +func CallPrivateRPC(inputJSON string) string { + return statusBackend.CallPrivateRPC(inputJSON) +} + +//CreateAccount is equivalent to creating an account from the command line, +// just modified to handle the function arg passing +func CreateAccount(password string) string { + address, pubKey, mnemonic, err := statusBackend.AccountManager().CreateAccount(password) + + errString := "" + if err != nil { + fmt.Fprintln(os.Stderr, err) + errString = err.Error() + } + + out := AccountInfo{ + Address: address, + PubKey: pubKey, + Mnemonic: mnemonic, + Error: errString, + } + outBytes, _ := json.Marshal(out) + return string(outBytes) +} + +//CreateChildAccount creates sub-account +func CreateChildAccount(parentAddress, password string) string { + address, pubKey, err := statusBackend.AccountManager().CreateChildAccount(parentAddress, password) + + errString := "" + if err != nil { + fmt.Fprintln(os.Stderr, err) + errString = err.Error() + } + + out := AccountInfo{ + Address: address, + PubKey: pubKey, + Error: errString, + } + outBytes, _ := json.Marshal(out) + return string(outBytes) +} + +//RecoverAccount re-creates master key using given details +func RecoverAccount(password, mnemonic string) string { + address, pubKey, err := statusBackend.AccountManager().RecoverAccount(password, mnemonic) + + errString := "" + if err != nil { + fmt.Fprintln(os.Stderr, err) + errString = err.Error() + } + + out := AccountInfo{ + Address: address, + PubKey: pubKey, + Mnemonic: (mnemonic), + Error: errString, + } + outBytes, _ := json.Marshal(out) + return string(outBytes) +} + +//VerifyAccountPassword verifies account password +func VerifyAccountPassword(keyStoreDir, address, password string) string { + _, err := statusBackend.AccountManager().VerifyAccountPassword(keyStoreDir, address, password) + return makeJSONResponse(err) +} + +//Login loads a key file (for a given address), tries to decrypt it using the password, to verify ownership +// if verified, purges all the previous identities from Whisper, and injects verified key as shh identity +func Login(address, password string) string { + err := statusBackend.SelectAccount(address, password) + return makeJSONResponse(err) +} + +//Logout is equivalent to clearing whisper identities +func Logout() string { + err := statusBackend.Logout() + return makeJSONResponse(err) +} + +// SignMessage unmarshals rpc params {data, address, password} and passes +// them onto backend.SignMessage +func SignMessage(rpcParams string) string { + var params personal.SignParams + err := json.Unmarshal([]byte(rpcParams), ¶ms) + if err != nil { + return prepareJSONResponseWithCode(nil, err, codeFailedParseParams) + } + result, err := statusBackend.SignMessage(params) + return prepareJSONResponse(result.String(), err) +} + +// Recover unmarshals rpc params {signDataString, signedData} and passes +// them onto backend. +func Recover(rpcParams string) string { + var params personal.RecoverParams + err := json.Unmarshal([]byte(rpcParams), ¶ms) + if err != nil { + return prepareJSONResponseWithCode(nil, err, codeFailedParseParams) + } + addr, err := statusBackend.Recover(params) + return prepareJSONResponse(addr.String(), err) +} + +// SendTransaction converts RPC args and calls backend.SendTransaction +func SendTransaction(txArgsJSON, password string) string { + var params transactions.SendTxArgs + err := json.Unmarshal([]byte(txArgsJSON), ¶ms) + if err != nil { + return prepareJSONResponseWithCode(nil, err, codeFailedParseParams) + } + hash, err := statusBackend.SendTransaction(params, password) + code := codeUnknown + if c, ok := errToCodeMap[err]; ok { + code = c + } + return prepareJSONResponseWithCode(hash.String(), err, code) +} + +//StartCPUProfile runs pprof for cpu +func StartCPUProfile(dataDir string) string { + err := profiling.StartCPUProfile(dataDir) + return makeJSONResponse(err) +} + +//StopCPUProfiling stops pprof for cpu +func StopCPUProfiling() string { //nolint: deadcode + err := profiling.StopCPUProfile() + return makeJSONResponse(err) +} + +//WriteHeapProfile starts pprof for heap +func WriteHeapProfile(dataDir string) string { //nolint: deadcode + err := profiling.WriteHeapFile(dataDir) + return makeJSONResponse(err) +} + +func makeJSONResponse(err error) string { + errString := "" + if err != nil { + fmt.Fprintln(os.Stderr, err) + errString = err.Error() + } + + out := APIResponse{ + Error: errString, + } + outBytes, _ := json.Marshal(out) + + return string(outBytes) +} + +// NotifyUsers sends push notifications by given tokens. +func NotifyUsers(message, payloadJSON, tokensArray string) (outCBytes string) { + var ( + err error + outBytes []byte + ) + errString := "" + + defer func() { + out := NotifyResult{ + Status: err == nil, + Error: errString, + } + + outBytes, err = json.Marshal(out) + if err != nil { + logger.Error("failed to marshal Notify output", "error", err) + outCBytes = makeJSONResponse(err) + return + } + + outCBytes = string(outBytes) + }() + + tokens, err := ParseJSONArray(tokensArray) + if err != nil { + errString = err.Error() + return + } + + var payload fcm.NotificationPayload + err = json.Unmarshal([]byte(payloadJSON), &payload) + if err != nil { + errString = err.Error() + return + } + + err = statusBackend.NotifyUsers(message, payload, tokens...) + if err != nil { + errString = err.Error() + return + } + + return +} + +// AddPeer adds an enode as a peer. +func AddPeer(enode string) string { + err := statusBackend.StatusNode().AddPeer(enode) + return makeJSONResponse(err) +} + +// ConnectionChange handles network state changes as reported +// by ReactNative (see https://facebook.github.io/react-native/docs/netinfo.html) +func ConnectionChange(typ string, expensive int) { + statusBackend.ConnectionChange(typ, expensive == 1) +} + +// AppStateChange handles app state changes (background/foreground). +func AppStateChange(state string) { + statusBackend.AppStateChange(state) +} + +// SetSignalEventCallback setup geth callback to notify about new signal +func SetSignalEventCallback(cb unsafe.Pointer) { + signal.SetSignalEventCallback(cb) +} diff --git a/mobile/types.go b/mobile/types.go new file mode 100644 index 00000000000..9ba89d19ee7 --- /dev/null +++ b/mobile/types.go @@ -0,0 +1,77 @@ +package status + +import ( + "bytes" + "fmt" + "strings" +) + +// APIResponse generic response from API. +type APIResponse struct { + Error string `json:"error"` +} + +// APIDetailedResponse represents a generic response +// with possible errors. +type APIDetailedResponse struct { + Status bool `json:"status"` + Message string `json:"message,omitempty"` + FieldErrors []APIFieldError `json:"field_errors,omitempty"` +} + +// Error string representation of APIDetailedResponse. +func (r APIDetailedResponse) Error() string { + buf := bytes.NewBufferString("") + + for _, err := range r.FieldErrors { + buf.WriteString(err.Error() + "\n") // nolint: gas + } + + return strings.TrimSpace(buf.String()) +} + +// APIFieldError represents a set of errors +// related to a parameter. +type APIFieldError struct { + Parameter string `json:"parameter,omitempty"` + Errors []APIError `json:"errors"` +} + +// Error string representation of APIFieldError. +func (e APIFieldError) Error() string { + if len(e.Errors) == 0 { + return "" + } + + buf := bytes.NewBufferString(fmt.Sprintf("Parameter: %s\n", e.Parameter)) + + for _, err := range e.Errors { + buf.WriteString(err.Error() + "\n") // nolint: gas + } + + return strings.TrimSpace(buf.String()) +} + +// APIError represents a single error. +type APIError struct { + Message string `json:"message"` +} + +// Error string representation of APIError. +func (e APIError) Error() string { + return fmt.Sprintf("message=%s", e.Message) +} + +// AccountInfo represents account's info. +type AccountInfo struct { + Address string `json:"address"` + PubKey string `json:"pubkey"` + Mnemonic string `json:"mnemonic"` + Error string `json:"error"` +} + +// NotifyResult is a JSON returned from notify message. +type NotifyResult struct { + Status bool `json:"status"` + Error string `json:"error,omitempty"` +} diff --git a/mobile/utils.go b/mobile/utils.go new file mode 100644 index 00000000000..ec56ef83dff --- /dev/null +++ b/mobile/utils.go @@ -0,0 +1,16 @@ +package status + +import ( + "encoding/json" +) + +// ParseJSONArray parses JSON array into Go array of string. +func ParseJSONArray(items string) ([]string, error) { + var parsedItems []string + err := json.Unmarshal([]byte(items), &parsedItems) + if err != nil { + return nil, err + } + + return parsedItems, nil +} From 94b9880848c80a07c1b2e76b682077df33e53361 Mon Sep 17 00:00:00 2001 From: Ivan Danyliuk Date: Wed, 22 Aug 2018 12:55:50 +0300 Subject: [PATCH 02/61] Update Makefile to use gomobile --- Makefile | 35 ++++++++++++++++------------------- README.md | 4 ++-- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 7576a5b69f0..914efd866ce 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: statusgo statusd-prune all test xgo clean help +.PHONY: statusgo statusd-prune all test xgo gomobile clean help .PHONY: statusgo-android statusgo-ios help: ##@other Show this help @@ -101,24 +101,17 @@ statusgo-linux: xgo ##@cross-compile Build status-go for Linux ./_assets/patches/patcher -b . -p geth-xgo -r @echo "Android cross compilation done." -statusgo-android: xgo ##@cross-compile Build status-go for Android - ./_assets/patches/patcher -b . -p geth-xgo - $(GOPATH)/bin/xgo --image $(XGOIMAGE) --go=$(GO) -out statusgo --dest=$(GOBIN) --targets=android-16/aar -v -tags '$(BUILD_TAGS)' $(BUILD_FLAGS) ./lib - ./_assets/patches/patcher -b . -p geth-xgo -r - @echo "Android cross compilation done." +## TODO(divan): rename statusgo-android-16.aar to Statusgo.aar +statusgo-android: gomobile ##@cross-compile Build status-go for Android + @echo "Building status-go for Android..." + @gomobile bind -target=android/arm -ldflags="-s -w" -o build/bin/statusgo-android-16.aar github.com/status-im/status-go/mobile + @echo "Android cross compilation done in build/bin/statusgo-android-16.aar" -statusgo-ios: xgo ##@cross-compile Build status-go for iOS - ./_assets/patches/patcher -b . -p geth-xgo - $(GOPATH)/bin/xgo --image $(XGOIMAGE) --go=$(GO) -out statusgo --dest=$(GOBIN) --targets=ios-9.3/framework -v -tags '$(BUILD_TAGS)' $(BUILD_FLAGS) ./lib - ./_assets/patches/patcher -b . -p geth-xgo -r - @echo "iOS framework cross compilation done." - -statusgo-ios-simulator: xgo ##@cross-compile Build status-go for iOS Simulator - @docker pull $(XGOIMAGEIOSSIM) - ./_assets/patches/patcher -b . -p geth-xgo - $(GOPATH)/bin/xgo --image $(XGOIMAGEIOSSIM) --go=$(GO) -out statusgo --dest=$(GOBIN) --targets=ios-9.3/framework -v -tags '$(BUILD_TAGS)' $(BUILD_FLAGS) ./lib - ./_assets/patches/patcher -b . -p geth-xgo -r - @echo "iOS framework cross compilation done." +## TODO(divan): rename statusgo-ios-9.3-framework to Statusgo.framework/ +statusgo-ios: gomobile ##@cross-compile Build status-go for iOS + @echo "Building status-go for iOS..." + @gomobile bind -target=ios -ldflags="-s -w" -o build/bin/statusgo-ios-9.3-framework github.com/status-im/status-go/mobile + @echo "iOS framework cross compilation done in build/bin/statusgo-ios-9.3-framework" statusgo-library: ##@cross-compile Build status-go as static library for current platform @echo "Building static library..." @@ -171,7 +164,11 @@ xgo: docker pull $(XGOIMAGE) go get github.com/karalabe/xgo -setup: dep-install lint-install mock-install ##@other Prepare project for first build +gomobile: + @echo "Installing gomobile..." + @go get -u golang.org/x/mobile/cmd/gomobile + +setup: dep-install lint-install mock-install gomobile ##@other Prepare project for first build mock-install: ##@other Install mocking tools go get -u github.com/golang/mock/mockgen diff --git a/README.md b/README.md index a1f84a14548..c497ec633b8 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ There are two main modes status-go can be built: Use following Makefile commands: - `make statusgo` (builds binary into `build/bin/statusd`) -- `make statusgo-android` (builds .aar file `build/android-16/aar`) -- `make statusgo-ios` and `make statusgo-ios-simulator` (builds iOS related artifacts in `build/os-9.3/framework`) +- `make statusgo-android` (builds .aar file `build/bin/Statusgo.aar`) +- `make statusgo-ios` (builds iOS framework in `build/bin/Statusgo.framework`) In order to build and use `status-go` directly from `status-react`, follow the instructions in https://wiki.status.im/Building_Status, under the '**Building Status with the checked-out version of status-go**' section. From 041990b4dcc6b87f0dd48f4c429093ba5bf2b1cf Mon Sep 17 00:00:00 2001 From: Ivan Danyliuk Date: Wed, 22 Aug 2018 15:32:46 +0300 Subject: [PATCH 03/61] Add README to mobile --- README.md | 4 ++-- mobile/README.md | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 mobile/README.md diff --git a/README.md b/README.md index c497ec633b8..90544bbc381 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ There are two main modes status-go can be built: Use following Makefile commands: - `make statusgo` (builds binary into `build/bin/statusd`) -- `make statusgo-android` (builds .aar file `build/bin/Statusgo.aar`) -- `make statusgo-ios` (builds iOS framework in `build/bin/Statusgo.framework`) +- `make statusgo-android` (builds .aar file `build/bin/statusgo-android-16.aar`) +- `make statusgo-ios` (builds iOS framework in `build/bin/statusgo-ios-9.3-framework`) In order to build and use `status-go` directly from `status-react`, follow the instructions in https://wiki.status.im/Building_Status, under the '**Building Status with the checked-out version of status-go**' section. diff --git a/mobile/README.md b/mobile/README.md new file mode 100644 index 00000000000..e60dee94287 --- /dev/null +++ b/mobile/README.md @@ -0,0 +1,27 @@ +# Mobile + +Package mobile implements [gomobile](https://github.com/golang/mobile) bindings for status-go. Current implementation servers as a drop-in replacement for `lib` package. + +# Usage + +For properly using this package, please refer to Makefile in the root of `status-go` directory. + +To manually build library, run following commands: + +### iOS + +``` +gomobile bind -v -target=ios -ldflags="-s -w" github.com/status-im/status-go/mobile +``` +This will produce `Status.framework` file in the current directory, which can be used in iOS project. + +### Android + +``` +gomobile bind -v -target=android -ldflags="-s -w" github.com/status-im/status-go/mobile +``` +This will generate `Status.aar` file in the current dir. + +# Notes + +See [https://github.com/golang/go/wiki/Mobile](https://github.com/golang/go/wiki/Mobile) for more information on `gomobile` usage. From 6fa22dbe372b7f5cdeb1cb22810b70bedf73ad70 Mon Sep 17 00:00:00 2001 From: Ivan Danyliuk Date: Wed, 31 Oct 2018 10:51:04 +0100 Subject: [PATCH 04/61] Fix library API --- mobile/status.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile/status.go b/mobile/status.go index 209c08bb71d..1e27bfcef00 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -25,7 +25,7 @@ var logger = log.New("package", "status-go/lib") //GenerateConfig for status node func GenerateConfig(datadir string, networkID int) string { - config, err := params.NewNodeConfig(datadir, "", params.FleetBeta, uint64(networkID)) + config, err := params.NewNodeConfig(datadir, uint64(networkID)) if err != nil { return makeJSONResponse(err) } @@ -40,7 +40,7 @@ func GenerateConfig(datadir string, networkID int) string { //StartNode - start Status node func StartNode(configJSON string) string { - config, err := params.LoadNodeConfig(configJSON) + config, err := params.NewConfigFromJSON(configJSON) if err != nil { return makeJSONResponse(err) } @@ -64,7 +64,7 @@ func StopNode() string { func ValidateNodeConfig(configJSON string) string { var resp APIDetailedResponse - _, err := params.LoadNodeConfig(configJSON) + _, err := params.NewConfigFromJSON(configJSON) // Convert errors to APIDetailedResponse switch err := err.(type) { From b178f2573133a5acedf9d672bf52d30f34afe28c Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 10:16:39 +0100 Subject: [PATCH 05/61] clean up Makefile --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8063bc1217e..a7992825269 100644 --- a/Makefile +++ b/Makefile @@ -200,11 +200,10 @@ generate: ##@other Regenerate assets and other auto-generated stuff go generate ./static ./static/migrations $(shell cd ./services/shhext/chat && exec protoc --go_out=. ./*.proto) -gomobile: - @echo "Installing gomobile..." - @go get -u golang.org/x/mobile/cmd/gomobile +setup: dep-install lint-install mock-install gomobile-install ##@other Prepare project for first build -setup: dep-install lint-install mock-install gomobile ##@other Prepare project for first build +gomobile-install: + go get -u golang.org/x/mobile/cmd/gomobile gen-install: go get -u github.com/jteeuwen/go-bindata/... From 606ddfdb4819d914eb2f24612c52855fce9d23bb Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 10:45:29 +0100 Subject: [PATCH 06/61] update missing exports --- Makefile | 14 ++--- mobile/status.go | 159 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 144 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index f0334bc8a31..4fb209092ce 100644 --- a/Makefile +++ b/Makefile @@ -121,17 +121,15 @@ statusgo-linux: xgo ##@cross-compile Build status-go for Linux ./_assets/patches/patcher -b . -p geth-xgo -r @echo "Android cross compilation done." -## TODO(divan): rename statusgo-android-16.aar to Statusgo.aar -statusgo-android: gomobile ##@cross-compile Build status-go for Android +statusgo-android: ##@cross-compile Build status-go for Android @echo "Building status-go for Android..." - @gomobile bind -target=android/arm -ldflags="-s -w" -o build/bin/statusgo-android-16.aar github.com/status-im/status-go/mobile - @echo "Android cross compilation done in build/bin/statusgo-android-16.aar" + @gomobile bind -target=android/arm -ldflags="-s -w" -o build/bin/statusgo.aar github.com/status-im/status-go/mobile + @echo "Android cross compilation done in build/bin/statusgo.aar" -## TODO(divan): rename statusgo-ios-9.3-framework to Statusgo.framework/ -statusgo-ios: gomobile ##@cross-compile Build status-go for iOS +statusgo-ios: ##@cross-compile Build status-go for iOS @echo "Building status-go for iOS..." - @gomobile bind -target=ios -ldflags="-s -w" -o build/bin/statusgo-ios-9.3-framework github.com/status-im/status-go/mobile - @echo "iOS framework cross compilation done in build/bin/statusgo-ios-9.3-framework" + @gomobile bind -target=ios -ldflags="-s -w" -o build/bin/Statusgo.framework github.com/status-im/status-go/mobile + @echo "iOS framework cross compilation done in build/bin/Statusgo.framework" statusgo-library: ##@cross-compile Build status-go as static library for current platform @echo "Building static library..." diff --git a/mobile/status.go b/mobile/status.go index 1e27bfcef00..2b09d4aea68 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -21,9 +21,9 @@ import ( var statusBackend = api.NewStatusBackend() // All general log messages in this package should be routed through this logger. -var logger = log.New("package", "status-go/lib") +var logger = log.New("package", "status-go/mobile") -//GenerateConfig for status node +// GenerateConfig for status node. func GenerateConfig(datadir string, networkID int) string { config, err := params.NewNodeConfig(datadir, uint64(networkID)) if err != nil { @@ -38,7 +38,7 @@ func GenerateConfig(datadir string, networkID int) string { return string(outBytes) } -//StartNode - start Status node +// StartNode starts the Ethereum Status node. func StartNode(configJSON string) string { config, err := params.NewConfigFromJSON(configJSON) if err != nil { @@ -54,13 +54,129 @@ func StartNode(configJSON string) string { return makeJSONResponse(nil) } -//StopNode - stop status node +// StopNode stops the Ethereum Status node. func StopNode() string { api.RunAsync(statusBackend.StopNode) return makeJSONResponse(nil) } -//ValidateNodeConfig validates config for status node +// CreateContactCode creates an X3DH bundle. +func CreateContactCode() string { + bundle, err := statusBackend.CreateContactCode() + if err != nil { + return makeJSONResponse(err) + } + + return bundle +} + +// ProcessContactCode processes an X3DH bundle. +// TODO(adam): it looks like the return should be error. +func ProcessContactCode(bundle string) string { + err := statusBackend.ProcessContactCode(bundle) + if err != nil { + return makeJSONResponse(err) + } + + return "" +} + +// ExtractIdentityFromContactCode extracts an identity from an X3DH bundle. +func ExtractIdentityFromContactCode(bundle string) string { + identity, err := statusBackend.ExtractIdentityFromContactCode(bundle) + if err != nil { + return makeJSONResponse(err) + } + + if err := statusBackend.ProcessContactCode(bundle); err != nil { + return makeJSONResponse(err) + } + + data, err := json.Marshal(struct { + Identity string `json:"identity"` + }{Identity: identity}) + if err != nil { + return makeJSONResponse(err) + } + + return string(data) +} + +// ExtractGroupMembershipSignatures extract public keys from tuples of content/signature. +func ExtractGroupMembershipSignatures(signaturePairsStr string) string { + var signaturePairs [][2]string + + if err := json.Unmarshal([]byte(signaturePairsStr), &signaturePairs); err != nil { + return makeJSONResponse(err) + } + + identities, err := statusBackend.ExtractGroupMembershipSignatures(signaturePairs) + if err != nil { + return makeJSONResponse(err) + } + + data, err := json.Marshal(struct { + Identities []string `json:"identities"` + }{Identities: identities}) + if err != nil { + return makeJSONResponse(err) + } + + return string(data) +} + +// SignGroupMembership signs a string containing group membership information. +func SignGroupMembership(content string) string { + signature, err := statusBackend.SignGroupMembership(content) + if err != nil { + return makeJSONResponse(err) + } + + data, err := json.Marshal(struct { + Signature string `json:"signature"` + }{Signature: signature}) + if err != nil { + return makeJSONResponse(err) + } + + return string(data) +} + +// EnableInstallation enables an installation for multi-device sync. +func EnableInstallation(installationID string) string { + err := statusBackend.EnableInstallation(installationID) + if err != nil { + return makeJSONResponse(err) + } + + data, err := json.Marshal(struct { + Response string `json:"response"` + }{Response: "ok"}) + if err != nil { + return makeJSONResponse(err) + } + + return string(data) +} + +// DisableInstallation disables an installation for multi-device sync. +func DisableInstallation(installationID string) string { + err := statusBackend.DisableInstallation(installationID) + if err != nil { + return makeJSONResponse(err) + } + + data, err := json.Marshal(struct { + Response string `json:"response"` + }{Response: "ok"}) + if err != nil { + return makeJSONResponse(err) + } + + return string(data) +} + +// ValidateNodeConfig validates config for the Status node. func ValidateNodeConfig(configJSON string) string { var resp APIDetailedResponse @@ -102,24 +218,24 @@ func ValidateNodeConfig(configJSON string) string { return string(respJSON) } -//ResetChainData remove chain data from data directory +// ResetChainData removes chain data from data directory. func ResetChainData() string { api.RunAsync(statusBackend.ResetChainData) return makeJSONResponse(nil) } -//CallRPC calls public APIs via RPC +// CallRPC calls public APIs via RPC. func CallRPC(inputJSON string) string { return statusBackend.CallRPC(inputJSON) } -//CallPrivateRPC calls both public and private APIs via RPC +// CallPrivateRPC calls both public and private APIs via RPC. func CallPrivateRPC(inputJSON string) string { return statusBackend.CallPrivateRPC(inputJSON) } -//CreateAccount is equivalent to creating an account from the command line, -// just modified to handle the function arg passing +// CreateAccount is equivalent to creating an account from the command line, +// just modified to handle the function arg passing. func CreateAccount(password string) string { address, pubKey, mnemonic, err := statusBackend.AccountManager().CreateAccount(password) @@ -139,7 +255,7 @@ func CreateAccount(password string) string { return string(outBytes) } -//CreateChildAccount creates sub-account +// CreateChildAccount creates sub-account. func CreateChildAccount(parentAddress, password string) string { address, pubKey, err := statusBackend.AccountManager().CreateChildAccount(parentAddress, password) @@ -158,7 +274,7 @@ func CreateChildAccount(parentAddress, password string) string { return string(outBytes) } -//RecoverAccount re-creates master key using given details +// RecoverAccount re-creates master key using given details. func RecoverAccount(password, mnemonic string) string { address, pubKey, err := statusBackend.AccountManager().RecoverAccount(password, mnemonic) @@ -178,27 +294,28 @@ func RecoverAccount(password, mnemonic string) string { return string(outBytes) } -//VerifyAccountPassword verifies account password +// VerifyAccountPassword verifies account password. func VerifyAccountPassword(keyStoreDir, address, password string) string { _, err := statusBackend.AccountManager().VerifyAccountPassword(keyStoreDir, address, password) return makeJSONResponse(err) } -//Login loads a key file (for a given address), tries to decrypt it using the password, to verify ownership -// if verified, purges all the previous identities from Whisper, and injects verified key as shh identity +// Login loads a key file (for a given address), tries to decrypt it using the password, +// to verify ownership if verified, purges all the previous identities from Whisper, +// and injects verified key as shh identity. func Login(address, password string) string { err := statusBackend.SelectAccount(address, password) return makeJSONResponse(err) } -//Logout is equivalent to clearing whisper identities +// Logout is equivalent to clearing whisper identities. func Logout() string { err := statusBackend.Logout() return makeJSONResponse(err) } -// SignMessage unmarshals rpc params {data, address, password} and passes -// them onto backend.SignMessage +// SignMessage unmarshals rpc params {data, address, password} and +// passes them onto backend.SignMessage. func SignMessage(rpcParams string) string { var params personal.SignParams err := json.Unmarshal([]byte(rpcParams), ¶ms) @@ -221,7 +338,7 @@ func Recover(rpcParams string) string { return prepareJSONResponse(addr.String(), err) } -// SendTransaction converts RPC args and calls backend.SendTransaction +// SendTransaction converts RPC args and calls backend.SendTransaction. func SendTransaction(txArgsJSON, password string) string { var params transactions.SendTxArgs err := json.Unmarshal([]byte(txArgsJSON), ¶ms) @@ -236,13 +353,13 @@ func SendTransaction(txArgsJSON, password string) string { return prepareJSONResponseWithCode(hash.String(), err, code) } -//StartCPUProfile runs pprof for cpu +// StartCPUProfile runs pprof for CPU. func StartCPUProfile(dataDir string) string { err := profiling.StartCPUProfile(dataDir) return makeJSONResponse(err) } -//StopCPUProfiling stops pprof for cpu +// StopCPUProfiling stops pprof for cpu. func StopCPUProfiling() string { //nolint: deadcode err := profiling.StopCPUProfile() return makeJSONResponse(err) From a628bd217c97f405fcad2ab31a0a8ed0164eb198 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 10:55:40 +0100 Subject: [PATCH 07/61] clean up --- Makefile | 23 +- _assets/build/xgo/base/Dockerfile | 10 - _assets/build/xgo/base/build.sh | 617 ------------------ _assets/build/xgo/ios-simulator/Dockerfile | 20 - _assets/build/xgo/ios-simulator/update_ios.sh | 64 -- _assets/ci/Jenkinsfile | 6 +- ...1-fix-duktapev3-missing-SIZE_MAX-def.patch | 20 - .../0002-remove-dashboard-collectData.patch | 153 ----- _assets/patches/geth-xgo/README.md | 20 - 9 files changed, 4 insertions(+), 929 deletions(-) delete mode 100644 _assets/build/xgo/base/Dockerfile delete mode 100644 _assets/build/xgo/base/build.sh delete mode 100644 _assets/build/xgo/ios-simulator/Dockerfile delete mode 100644 _assets/build/xgo/ios-simulator/update_ios.sh delete mode 100644 _assets/patches/geth-xgo/0001-fix-duktapev3-missing-SIZE_MAX-def.patch delete mode 100644 _assets/patches/geth-xgo/0002-remove-dashboard-collectData.patch delete mode 100644 _assets/patches/geth-xgo/README.md diff --git a/Makefile b/Makefile index 4fb209092ce..f83920f457f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: statusgo statusd-prune all test xgo gomobile clean help +.PHONY: statusgo statusd-prune all test clean help .PHONY: statusgo-android statusgo-ios RELEASE_TAG := $(shell cat VERSION) @@ -42,11 +42,6 @@ BUILD_FLAGS ?= $(shell echo "-ldflags '\ -X github.com/status-im/status-go/params.GitCommit=$(GIT_COMMIT) \ -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=$(ENABLE_METRICS)'") -XGO_GO ?= latest -XGOVERSION ?= 1.10.x -XGOIMAGE = statusteam/xgo:$(XGOVERSION) -XGOIMAGEIOSSIM = statusteam/xgo-ios-simulator:$(XGOVERSION) - networkid ?= StatusChain gotest_extraflags = @@ -115,12 +110,6 @@ statusgo-cross: statusgo-android statusgo-ios @echo "Full cross compilation done." @ls -ld $(GOBIN)/statusgo-* -statusgo-linux: xgo ##@cross-compile Build status-go for Linux - ./_assets/patches/patcher -b . -p geth-xgo - $(GOPATH)/bin/xgo --image $(XGOIMAGE) --go=$(XGO_GO) -out statusgo --dest=$(GOBIN) --targets=linux/amd64 -v -tags '$(BUILD_TAGS)' $(BUILD_FLAGS) ./cmd/statusd - ./_assets/patches/patcher -b . -p geth-xgo -r - @echo "Android cross compilation done." - statusgo-android: ##@cross-compile Build status-go for Android @echo "Building status-go for Android..." @gomobile bind -target=android/arm -ldflags="-s -w" -o build/bin/statusgo.aar github.com/status-im/status-go/mobile @@ -187,16 +176,6 @@ endif docker push $(BOOTNODE_IMAGE_NAME):latest docker push $(DOCKER_IMAGE_NAME):latest -xgo-docker-images: ##@docker Build xgo docker images - @echo "Building xgo docker images..." - docker build _assets/build/xgo/base -t $(XGOIMAGE) - docker build _assets/build/xgo/ios-simulator -t $(XGOIMAGEIOSSIM) - -xgo: - docker pull $(XGOIMAGE) - go get github.com/status-im/xgo - mkdir -p $(GOBIN) - install-os-dependencies: _assets/scripts/install_deps.sh diff --git a/_assets/build/xgo/base/Dockerfile b/_assets/build/xgo/base/Dockerfile deleted file mode 100644 index 8ee76b64a9a..00000000000 --- a/_assets/build/xgo/base/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM karalabe/xgo-1.10.x - -VOLUME [ "/build", "/deps-cache" ] - -# Inject the container entry point, the build script (patched for Status bindings conditional builds of C code) -ADD build.sh /build.sh -ENV BUILD /build.sh -RUN chmod +x $BUILD - -ENTRYPOINT ["/build.sh"] diff --git a/_assets/build/xgo/base/build.sh b/_assets/build/xgo/base/build.sh deleted file mode 100644 index e63a0336df4..00000000000 --- a/_assets/build/xgo/base/build.sh +++ /dev/null @@ -1,617 +0,0 @@ -#!/bin/bash -# -# Contains the main cross compiler, that individually sets up each target build -# platform, compiles all the C dependencies, then build the requested executable -# itself. -# -# Usage: build.sh -# -# Needed environment variables: -# REPO_REMOTE - Optional VCS remote if not the primary repository is needed -# REPO_BRANCH - Optional VCS branch to use, if not the master branch -# DEPS - Optional list of C dependency packages to build -# ARGS - Optional arguments to pass to C dependency configure scripts -# PACK - Optional sub-package, if not the import path is being built -# OUT - Optional output prefix to override the package name -# FLAG_V - Optional verbosity flag to set on the Go builder -# FLAG_X - Optional flag to print the build progress commands -# FLAG_RACE - Optional race flag to set on the Go builder -# FLAG_TAGS - Optional tag flag to set on the Go builder -# FLAG_LDFLAGS - Optional ldflags flag to set on the Go builder -# FLAG_BUILDMODE - Optional buildmode flag to set on the Go builder -# TARGETS - Comma separated list of build targets to compile for -# GO_VERSION - Bootstrapped version of Go to disable uncupported targets -# EXT_GOPATH - GOPATH elements mounted from the host filesystem - -# Define a function that figures out the binary extension -function extension { - if [ "$FLAG_BUILDMODE" == "archive" ] || [ "$FLAG_BUILDMODE" == "c-archive" ]; then - if [ "$1" == "windows" ]; then - echo ".lib" - else - echo ".a" - fi - elif [ "$FLAG_BUILDMODE" == "shared" ] || [ "$FLAG_BUILDMODE" == "c-shared" ]; then - if [ "$1" == "windows" ]; then - echo ".dll" - elif [ "$1" == "darwin" ] || [ "$1" == "ios" ]; then - echo ".dylib" - else - echo ".so" - fi - else - if [ "$1" == "windows" ]; then - echo ".exe" - fi - fi -} - -# Either set a local build environment, or pull any remote imports -if [ "$EXT_GOPATH" != "" ]; then - # If local builds are requested, inject the sources - echo "Building locally $1..." - export GOPATH=$GOPATH:$EXT_GOPATH - set -e - - # Find and change into the package folder - cd `go list -e -f {{.Dir}} $1` - export GOPATH=$GOPATH:`pwd`/Godeps/_workspace -else - # Inject all possible Godep paths to short circuit go gets - GOPATH_ROOT=$GOPATH/src - IMPORT_PATH=$1 - while [ "$IMPORT_PATH" != "." ]; do - export GOPATH=$GOPATH:$GOPATH_ROOT/$IMPORT_PATH/Godeps/_workspace - IMPORT_PATH=`dirname $IMPORT_PATH` - done - - # Otherwise download the canonical import path (may fail, don't allow failures beyond) - echo "Fetching main repository $1..." - go get -v -d $1 - set -e - - cd $GOPATH_ROOT/$1 - - # Switch over the code-base to another checkout if requested - if [ "$REPO_REMOTE" != "" ] || [ "$REPO_BRANCH" != "" ]; then - # Detect the version control system type - IMPORT_PATH=$1 - while [ "$IMPORT_PATH" != "." ] && [ "$REPO_TYPE" == "" ]; do - if [ -d "$GOPATH_ROOT/$IMPORT_PATH/.git" ]; then - REPO_TYPE="git" - elif [ -d "$GOPATH_ROOT/$IMPORT_PATH/.hg" ]; then - REPO_TYPE="hg" - fi - IMPORT_PATH=`dirname $IMPORT_PATH` - done - - if [ "$REPO_TYPE" == "" ]; then - echo "Unknown version control system type, cannot switch remotes and branches." - exit -1 - fi - # If we have a valid VCS, execute the switch operations - if [ "$REPO_REMOTE" != "" ]; then - echo "Switching over to remote $REPO_REMOTE..." - if [ "$REPO_TYPE" == "git" ]; then - git remote set-url origin $REPO_REMOTE - git fetch --all - git reset --hard origin/HEAD - git clean -dxf - elif [ "$REPO_TYPE" == "hg" ]; then - echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc - hg pull - fi - fi - if [ "$REPO_BRANCH" != "" ]; then - echo "Switching over to branch $REPO_BRANCH..." - if [ "$REPO_TYPE" == "git" ]; then - git reset --hard origin/$REPO_BRANCH - git clean -dxf - elif [ "$REPO_TYPE" == "hg" ]; then - hg checkout $REPO_BRANCH - fi - fi - fi -fi - -# Download all the C dependencies -mkdir /deps -DEPS=($DEPS) && for dep in "${DEPS[@]}"; do - if [ "${dep##*.}" == "tar" ]; then cat "/deps-cache/`basename $dep`" | tar -C /deps -x; fi - if [ "${dep##*.}" == "gz" ]; then cat "/deps-cache/`basename $dep`" | tar -C /deps -xz; fi - if [ "${dep##*.}" == "bz2" ]; then cat "/deps-cache/`basename $dep`" | tar -C /deps -xj; fi -done - -DEPS_ARGS=($ARGS) - -# Save the contents of the pre-build /usr/local folder for post cleanup -USR_LOCAL_CONTENTS=`ls /usr/local` - -# Configure some global build parameters -NAME=`basename $1/$PACK` -if [ "$OUT" != "" ]; then - NAME=$OUT -fi - -if [ "$FLAG_V" == "true" ]; then V=-v; fi -if [ "$FLAG_X" == "true" ]; then X=-x; fi -if [ "$FLAG_RACE" == "true" ]; then R=-race; fi -if [ "$FLAG_TAGS" != "" ]; then T=(--tags "$FLAG_TAGS"); fi -if [ "$FLAG_LDFLAGS" != "" ]; then LD="$FLAG_LDFLAGS"; fi - -if [ "$FLAG_BUILDMODE" != "" ] && [ "$FLAG_BUILDMODE" != "default" ]; then BM="--buildmode=$FLAG_BUILDMODE"; fi - -# If no build targets were specified, inject a catch all wildcard -if [ "$TARGETS" == "" ]; then - TARGETS="./." -fi - -# Build for each requested platform individually -for TARGET in $TARGETS; do - # Split the target into platform and architecture - XGOOS=`echo $TARGET | cut -d '/' -f 1` - XGOARCH=`echo $TARGET | cut -d '/' -f 2` - - # Check and build for Android targets - if ([ $XGOOS == "." ] || [[ $XGOOS == android* ]]); then - # Split the platform version and configure the linker options - PLATFORM=`echo $XGOOS | cut -d '-' -f 2` - if [ "$PLATFORM" == "" ] || [ "$PLATFORM" == "." ] || [ "$PLATFORM" == "android" ]; then - PLATFORM=16 # Jelly Bean 4.0.0 - fi - CGO_STATUS_IM="-D ANDROID_DEPLOYMENT" - if [ "$PLATFORM" -ge 16 ]; then - CGO_CCPIE="-fPIE" - CGO_LDPIE="-fPIE" - EXT_LDPIE="-extldflags=-pie" - else - unset CGO_CCPIE CGO_LDPIE EXT_LDPIE - fi - mkdir -p /build-android-aar - - # Iterate over the requested architectures, bootstrap and build - if [ $XGOARCH == "." ] || [ $XGOARCH == "arm" ] || [ $XGOARCH == "aar" ]; then - if [ "$GO_VERSION" -lt 150 ]; then - echo "Go version too low, skipping android-$PLATFORM/arm..." - else - # Include a linker workaround for pre Go 1.6 releases - if [ "$GO_VERSION" -lt 160 ]; then - EXT_LDAMD="-extldflags=-Wl,--allow-multiple-definition" - fi - - echo "Assembling toolchain for android-$PLATFORM/arm..." - $ANDROID_NDK_ROOT/build/tools/make-standalone-toolchain.sh --ndk-dir=$ANDROID_NDK_ROOT --install-dir=/usr/$ANDROID_CHAIN_ARM --toolchain=$ANDROID_CHAIN_ARM --arch=arm > /dev/null 2>&1 - - echo "Bootstrapping android-$PLATFORM/arm..." - CC=arm-linux-androideabi-gcc GOOS=android GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go install std - - echo "Compiling for android-$PLATFORM/arm..." - CC=arm-linux-androideabi-gcc CXX=arm-linux-androideabi-g++ HOST=arm-linux-androideabi PREFIX=/usr/$ANDROID_CHAIN_ARM/arm-linux-androideabi $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/$ANDROID_CHAIN_ARM/arm-linux-androideabi/lib/pkgconfig - - if [ $XGOARCH == "." ] || [ $XGOARCH == "arm" ]; then - CC=arm-linux-androideabi-gcc CXX=arm-linux-androideabi-g++ GOOS=android GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE" CGO_CXXFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CGO_CFLAGS="-D ANDROID_DEPLOYMENT" CC=arm-linux-androideabi-gcc CXX=arm-linux-androideabi-g++ GOOS=android GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE $CGO_STATUS_IM" CGO_CXXFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go build $V $X "${T[@]}" --ldflags="$V $EXT_LDPIE $EXT_LDAMD $LD" $BM -o "/build/$NAME-android-$PLATFORM-arm`extension android`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "aar" ]; then - CC=arm-linux-androideabi-gcc CXX=arm-linux-androideabi-g++ GOOS=android GOARCH=arm GOARM=7 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=arm-linux-androideabi-gcc CXX=arm-linux-androideabi-g++ GOOS=android GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${T[@]}" --ldflags="$V $EXT_LDAMD $LD" --buildmode=c-shared -o "/build-android-aar/$NAME-android-$PLATFORM-arm.so" ./$PACK - fi - fi - fi - if [ "$GO_VERSION" -lt 160 ]; then - echo "Go version too low, skipping android-$PLATFORM/386,arm64..." - else - if [ "$PLATFORM" -ge 9 ] && ([ $XGOARCH == "." ] || [ $XGOARCH == "386" ] || [ $XGOARCH == "aar" ]); then - echo "Assembling toolchain for android-$PLATFORM/386..." - $ANDROID_NDK_ROOT/build/tools/make-standalone-toolchain.sh --ndk-dir=$ANDROID_NDK_ROOT --install-dir=/usr/$ANDROID_CHAIN_386 --toolchain=$ANDROID_CHAIN_386 --arch=x86 > /dev/null 2>&1 - - echo "Bootstrapping android-$PLATFORM/386..." - CC=i686-linux-android-gcc GOOS=android GOARCH=386 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go install std - - echo "Compiling for android-$PLATFORM/386..." - CC=i686-linux-android-gcc CXX=i686-linux-android-g++ HOST=i686-linux-android PREFIX=/usr/$ANDROID_CHAIN_386/i686-linux-android $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/$ANDROID_CHAIN_386/i686-linux-android/lib/pkgconfig - - if [ $XGOARCH == "." ] || [ $XGOARCH == "386" ]; then - CC=i686-linux-android-gcc CXX=i686-linux-android-g++ GOOS=android GOARCH=386 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE" CGO_CXXFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=i686-linux-android-gcc CXX=i686-linux-android-g++ GOOS=android GOARCH=386 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE $CGO_STATUS_IM" CGO_CXXFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go build $V $X "${T[@]}" --ldflags="$V $EXT_LDPIE $LD" $BM -o "/build/$NAME-android-$PLATFORM-386`extension android`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "aar" ]; then - CC=i686-linux-android-gcc CXX=i686-linux-android-g++ GOOS=android GOARCH=386 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=i686-linux-android-gcc CXX=i686-linux-android-g++ GOOS=android GOARCH=386 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${T[@]}" --ldflags="$V $LD" --buildmode=c-shared -o "/build-android-aar/$NAME-android-$PLATFORM-386.so" ./$PACK - fi - fi - if [ "$PLATFORM" -ge 21 ] && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm64" ] || [ $XGOARCH == "aar" ]); then - echo "Assembling toolchain for android-$PLATFORM/arm64..." - $ANDROID_NDK_ROOT/build/tools/make-standalone-toolchain.sh --ndk-dir=$ANDROID_NDK_ROOT --install-dir=/usr/$ANDROID_CHAIN_ARM64 --toolchain=$ANDROID_CHAIN_ARM64 --arch=arm64 > /dev/null 2>&1 - - echo "Bootstrapping android-$PLATFORM/arm64..." - CC=aarch64-linux-android-gcc GOOS=android GOARCH=arm64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go install std - - echo "Compiling for android-$PLATFORM/arm64..." - CC=aarch64-linux-android-gcc CXX=aarch64-linux-android-g++ HOST=aarch64-linux-android PREFIX=/usr/$ANDROID_CHAIN_ARM64/aarch64-linux-android $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/$ANDROID_CHAIN_ARM64/aarch64-linux-android/lib/pkgconfig - - if [ $XGOARCH == "." ] || [ $XGOARCH == "arm64" ]; then - CC=aarch64-linux-android-gcc CXX=aarch64-linux-android-g++ GOOS=android GOARCH=arm64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE" CGO_CXXFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=aarch64-linux-android-gcc CXX=aarch64-linux-android-g++ GOOS=android GOARCH=arm64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_CCPIE $CGO_STATUS_IM" CGO_CXXFLAGS="$CGO_CCPIE" CGO_LDFLAGS="$CGO_LDPIE" go build $V $X "${T[@]}" --ldflags="$V $EXT_LDPIE $LD" $BM -o "/build/$NAME-android-$PLATFORM-arm64`extension android`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "aar" ]; then - CC=aarch64-linux-android-gcc CXX=aarch64-linux-android-g++ GOOS=android GOARCH=arm64 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=aarch64-linux-android-gcc CXX=aarch64-linux-android-g++ GOOS=android GOARCH=arm64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${T[@]}" --ldflags="$V $LD" --buildmode=c-shared -o "/build-android-aar/$NAME-android-$PLATFORM-arm64.so" ./$PACK - fi - fi - fi - unset CGO_STATUS_IM # good to let that extra var go away - # Assemble the Android Archive from the built shared libraries - if [ $XGOARCH == "." ] || [ $XGOARCH == "aar" ]; then - title=${NAME^} - archive=/build/$NAME-android-$PLATFORM-aar - bundle=/build/$NAME-android-$PLATFORM.aar - - # Generate the Java import path based on the Go one - package=`go list ./$PACK | tr '-' '_'` - package=$(for p in `echo ${package//\// }`; do echo $p | awk 'BEGIN{FS="."}{for (i=NF; i>0; i--){printf "%s.", $i;}}'; done | sed 's/.$//') - package=${package%.*} - - # Create a fresh empty Android archive - rm -rf $archive $bundle - mkdir -p $archive - - echo -e "\n \n" > $archive/AndroidManifest.xml - mkdir -p $archive/res - touch $archive/R.txt - - # Generate the JNI wrappers automatically with SWIG - jni=`mktemp -d` - header=`find /build-android-aar | grep '\.h$' | head -n 1` - if [ "$header" == "" ]; then - echo "No API C header specified, skipping android-$PLATFORM/aar..." - else - cp $header $jni/$NAME.h - sed -i -e 's|__complex|complex|g' $jni/$NAME.h - sed -i -e 's|_Complex|complex|g' $jni/$NAME.h - echo -e "%module $title\n%{\n#include \"$NAME.h\"\n%}\n%pragma(java) jniclasscode=%{\nstatic {\nSystem.loadLibrary(\"$NAME\");\n}\n%}\n%include \"$NAME.h\"" > $jni/$NAME.i - - mkdir -p $jni/${package//.//} - swig -java -package $package -outdir $jni/${package//.//} $jni/$NAME.i - - # Assemble the Go static libraries and the JNI interface into shared libraries - for lib in `find /build-android-aar | grep '\.so$'`; do - if [[ "$lib" = *-arm.so ]]; then cc=arm-linux-androideabi-gcc; abi="armeabi-v7a"; fi - if [[ "$lib" = *-arm64.so ]]; then cc=aarch64-linux-android-gcc; abi="arm64-v8a"; fi - if [[ "$lib" = *-386.so ]]; then cc=i686-linux-android-gcc; abi="x86"; fi - - mkdir -p $archive/jni/$abi - cp ${lib%.*}.h $jni/${NAME}.h - cp $lib $archive/jni/$abi/lib${NAME}raw.so - (cd $archive/jni/$abi && $cc -shared -fPIC -o lib${NAME}.so -I"$ANDROID_NDK_LIBC/include" -I"$ANDROID_NDK_LIBC/libs/$abi/include" -I"$jni" lib${NAME}raw.so $jni/${NAME}_wrap.c) - done - - # Compile the Java wrapper and assemble into a .jar file - mkdir -p $jni/build - javac -target 1.7 -source 1.7 -cp . -d $jni/build $jni/${package//.//}/*.java - (cd $jni/build && jar cvf $archive/classes.jar *) - - # Finally assemble the archive contents into an .aar and clean up - (cd $archive && zip -r $bundle *) - rm -rf $jni $archive - fi - - # Fix up permissions on bundle file - chown $UID:$GID $bundle - fi - # Clean up the android builds, toolchains and runtimes - rm -rf /build-android-aar - rm -rf /usr/local/go/pkg/android_* - rm -rf /usr/$ANDROID_CHAIN_ARM /usr/$ANDROID_CHAIN_ARM64 /usr/$ANDROID_CHAIN_386 - fi - # Check and build for Linux targets - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "amd64" ]); then - echo "Compiling for linux/amd64..." - HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS /deps ${DEPS_ARGS[@]} - GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $X "${T[@]}" --ldflags="$V $LD" $R $BM -o "/build/$NAME-linux-amd64$R`extension linux`" ./$PACK - fi - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "386" ]); then - echo "Compiling for linux/386..." - HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS /deps ${DEPS_ARGS[@]} - GOOS=linux GOARCH=386 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-linux-386`extension linux`" ./$PACK - fi - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm" ] || [ $XGOARCH == "arm-5" ]); then - if [ "$GO_VERSION" -ge 150 ]; then - echo "Bootstrapping linux/arm-5..." - CC=arm-linux-gnueabi-gcc-5 GOOS=linux GOARCH=arm GOARM=5 CGO_ENABLED=1 CGO_CFLAGS="-march=armv5" CGO_CXXFLAGS="-march=armv5" go install std - fi - echo "Compiling for linux/arm-5..." - CC=arm-linux-gnueabi-gcc-5 CXX=arm-linux-gnueabi-g++-5 HOST=arm-linux-gnueabi PREFIX=/usr/arm-linux-gnueabi CFLAGS="-march=armv5" CXXFLAGS="-march=armv5" $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/arm-linux-gnueabi/lib/pkgconfig - - CC=arm-linux-gnueabi-gcc-5 CXX=arm-linux-gnueabi-g++-5 GOOS=linux GOARCH=arm GOARM=5 CGO_ENABLED=1 CGO_CFLAGS="-march=armv5" CGO_CXXFLAGS="-march=armv5" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=arm-linux-gnueabi-gcc-5 CXX=arm-linux-gnueabi-g++-5 GOOS=linux GOARCH=arm GOARM=5 CGO_ENABLED=1 CGO_CFLAGS="-march=armv5" CGO_CXXFLAGS="-march=armv5" go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-linux-arm-5`extension linux`" ./$PACK - if [ "$GO_VERSION" -ge 150 ]; then - echo "Cleaning up Go runtime for linux/arm-5..." - rm -rf /usr/local/go/pkg/linux_arm - fi - fi - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm-6" ]); then - if [ "$GO_VERSION" -lt 150 ]; then - echo "Go version too low, skipping linux/arm-6..." - else - echo "Bootstrapping linux/arm-6..." - CC=arm-linux-gnueabi-gcc-5 GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=1 CGO_CFLAGS="-march=armv6" CGO_CXXFLAGS="-march=armv6" go install std - - echo "Compiling for linux/arm-6..." - CC=arm-linux-gnueabi-gcc-5 CXX=arm-linux-gnueabi-g++-5 HOST=arm-linux-gnueabi PREFIX=/usr/arm-linux-gnueabi CFLAGS="-march=armv6" CXXFLAGS="-march=armv6" $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/arm-linux-gnueabi/lib/pkgconfig - - CC=arm-linux-gnueabi-gcc-5 CXX=arm-linux-gnueabi-g++-5 GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=1 CGO_CFLAGS="-march=armv6" CGO_CXXFLAGS="-march=armv6" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=arm-linux-gnueabi-gcc-5 CXX=arm-linux-gnueabi-g++-5 GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=1 CGO_CFLAGS="-march=armv6" CGO_CXXFLAGS="-march=armv6" go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-linux-arm-6`extension linux`" ./$PACK - - echo "Cleaning up Go runtime for linux/arm-6..." - rm -rf /usr/local/go/pkg/linux_arm - fi - fi - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm-7" ]); then - if [ "$GO_VERSION" -lt 150 ]; then - echo "Go version too low, skipping linux/arm-7..." - else - echo "Bootstrapping linux/arm-7..." - CC=arm-linux-gnueabihf-gcc-5 GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="-march=armv7-a" CGO_CXXFLAGS="-march=armv7-a" go install std - - echo "Compiling for linux/arm-7..." - CC=arm-linux-gnueabihf-gcc-5 CXX=arm-linux-gnueabihf-g++-5 HOST=arm-linux-gnueabihf PREFIX=/usr/arm-linux-gnueabihf CFLAGS="-march=armv7-a -fPIC" CXXFLAGS="-march=armv7-a -fPIC" $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/arm-linux-gnueabihf/lib/pkgconfig - - CC=arm-linux-gnueabihf-gcc-5 CXX=arm-linux-gnueabihf-g++-5 GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="-march=armv7-a -fPIC" CGO_CXXFLAGS="-march=armv7-a -fPIC" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=arm-linux-gnueabihf-gcc-5 CXX=arm-linux-gnueabihf-g++-5 GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="-march=armv7-a -fPIC" CGO_CXXFLAGS="-march=armv7-a -fPIC" go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-linux-arm-7`extension linux`" ./$PACK - - echo "Cleaning up Go runtime for linux/arm-7..." - rm -rf /usr/local/go/pkg/linux_arm - fi - fi - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm64" ]); then - if [ "$GO_VERSION" -lt 150 ]; then - echo "Go version too low, skipping linux/arm64..." - else - echo "Compiling for linux/arm64..." - CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 HOST=aarch64-linux-gnu PREFIX=/usr/aarch64-linux-gnu $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/aarch64-linux-gnu/lib/pkgconfig - - CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 GOOS=linux GOARCH=arm64 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 GOOS=linux GOARCH=arm64 CGO_ENABLED=1 go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-linux-arm64`extension linux`" ./$PACK - fi - fi - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "mips64" ]); then - if [ "$GO_VERSION" -lt 170 ]; then - echo "Go version too low, skipping linux/mips64..." - else - echo "Compiling for linux/mips64..." - CC=mips64-linux-gnuabi64-gcc-5 CXX=mips64-linux-gnuabi64-g++-5 HOST=mips64-linux-gnuabi64 PREFIX=/usr/mips64-linux-gnuabi64 $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/mips64-linux-gnuabi64/lib/pkgconfig - - CC=mips64-linux-gnuabi64-gcc-5 CXX=mips64-linux-gnuabi64-g++-5 GOOS=linux GOARCH=mips64 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=mips64-linux-gnuabi64-gcc-5 CXX=mips64-linux-gnuabi64-g++-5 GOOS=linux GOARCH=mips64 CGO_ENABLED=1 go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-linux-mips64`extension linux`" ./$PACK - fi - fi - if ([ $XGOOS == "." ] || [ $XGOOS == "linux" ]) && ([ $XGOARCH == "." ] || [ $XGOARCH == "mips64le" ]); then - if [ "$GO_VERSION" -lt 170 ]; then - echo "Go version too low, skipping linux/mips64le..." - else - echo "Compiling for linux/mips64le..." - CC=mips64el-linux-gnuabi64-gcc-5 CXX=mips64el-linux-gnuabi64-g++-5 HOST=mips64el-linux-gnuabi64 PREFIX=/usr/mips64el-linux-gnuabi64 $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/mips64le-linux-gnuabi64/lib/pkgconfig - - CC=mips64el-linux-gnuabi64-gcc-5 CXX=mips64el-linux-gnuabi64-g++-5 GOOS=linux GOARCH=mips64le CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=mips64el-linux-gnuabi64-gcc-5 CXX=mips64el-linux-gnuabi64-g++-5 GOOS=linux GOARCH=mips64le CGO_ENABLED=1 go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-linux-mips64le`extension linux`" ./$PACK - fi - fi - # Check and build for Windows targets - if [ $XGOOS == "." ] || [[ $XGOOS == windows* ]]; then - # Split the platform version and configure the Windows NT version - PLATFORM=`echo $XGOOS | cut -d '-' -f 2` - if [ "$PLATFORM" == "" ] || [ "$PLATFORM" == "." ] || [ "$PLATFORM" == "windows" ]; then - PLATFORM=4.0 # Windows NT - fi - - MAJOR=`echo $PLATFORM | cut -d '.' -f 1` - if [ "${PLATFORM/.}" != "$PLATFORM" ] ; then - MINOR=`echo $PLATFORM | cut -d '.' -f 2` - fi - CGO_NTDEF="-D_WIN32_WINNT=0x`printf "%02d" $MAJOR``printf "%02d" $MINOR`" - - # Build the requested windows binaries - if [ $XGOARCH == "." ] || [ $XGOARCH == "amd64" ]; then - echo "Compiling for windows-$PLATFORM/amd64..." - CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/x86_64-w64-mingw32/lib/pkgconfig - - CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_NTDEF" CGO_CXXFLAGS="$CGO_NTDEF" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_NTDEF" CGO_CXXFLAGS="$CGO_NTDEF" go build $V $X "${T[@]}" --ldflags="$V $LD" $R $BM -o "/build/$NAME-windows-$PLATFORM-amd64$R`extension windows`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "386" ]; then - echo "Compiling for windows-$PLATFORM/386..." - CC=i686-w64-mingw32-gcc-posix CXX=i686-w64-mingw32-g++-posix HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps ${DEPS_ARGS[@]} - export PKG_CONFIG_PATH=/usr/i686-w64-mingw32/lib/pkgconfig - - CC=i686-w64-mingw32-gcc-posix CXX=i686-w64-mingw32-g++-posix GOOS=windows GOARCH=386 CGO_ENABLED=1 CGO_CFLAGS="$CGO_NTDEF" CGO_CXXFLAGS="$CGO_NTDEF" go get $V $X "${T[@]}" --ldflags="$V $LD" -d ./$PACK - CC=i686-w64-mingw32-gcc-posix CXX=i686-w64-mingw32-g++-posix GOOS=windows GOARCH=386 CGO_ENABLED=1 CGO_CFLAGS="$CGO_NTDEF" CGO_CXXFLAGS="$CGO_NTDEF" go build $V $X "${T[@]}" --ldflags="$V $LD" $BM -o "/build/$NAME-windows-$PLATFORM-386`extension windows`" ./$PACK - fi - fi - # Check and build for OSX targets - if [ $XGOOS == "." ] || [[ $XGOOS == darwin* ]]; then - # Split the platform version and configure the deployment target - PLATFORM=`echo $XGOOS | cut -d '-' -f 2` - if [ "$PLATFORM" == "" ] || [ "$PLATFORM" == "." ] || [ "$PLATFORM" == "darwin" ]; then - PLATFORM=10.6 # OS X Snow Leopard - fi - export MACOSX_DEPLOYMENT_TARGET=$PLATFORM - - # Strip symbol table below Go 1.6 to prevent DWARF issues - LDSTRIP="" - if [ "$GO_VERSION" -lt 160 ]; then - LDSTRIP="-s" - fi - # Build the requested darwin binaries - if [ $XGOARCH == "." ] || [ $XGOARCH == "amd64" ]; then - echo "Compiling for darwin-$PLATFORM/amd64..." - CC=o64-clang CXX=o64-clang++ HOST=x86_64-apple-darwin15 PREFIX=/usr/local $BUILD_DEPS /deps ${DEPS_ARGS[@]} - CC=o64-clang CXX=o64-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$LDSTRIP $V $LD" -d ./$PACK - CC=o64-clang CXX=o64-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $X "${T[@]}" --ldflags="$LDSTRIP $V $LD" $R $BM -o "/build/$NAME-darwin-$PLATFORM-amd64$R`extension darwin`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "386" ]; then - echo "Compiling for darwin-$PLATFORM/386..." - CC=o32-clang CXX=o32-clang++ HOST=i386-apple-darwin15 PREFIX=/usr/local $BUILD_DEPS /deps ${DEPS_ARGS[@]} - CC=o32-clang CXX=o32-clang++ GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get $V $X "${T[@]}" --ldflags="$LDSTRIP $V $LD" -d ./$PACK - CC=o32-clang CXX=o32-clang++ GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V $X "${T[@]}" --ldflags="$LDSTRIP $V $LD" $BM -o "/build/$NAME-darwin-$PLATFORM-386`extension darwin`" ./$PACK - fi - # Remove any automatically injected deployment target vars - unset MACOSX_DEPLOYMENT_TARGET - fi - # Check and build for iOS targets - if [ $XGOOS == "." ] || [[ $XGOOS == ios* ]]; then - # Split the platform version and configure the deployment target - PLATFORM=`echo $XGOOS | cut -d '-' -f 2` - if [ "$PLATFORM" == "" ] || [ "$PLATFORM" == "." ] || [ "$PLATFORM" == "ios" ]; then - PLATFORM=5.0 # first iPad and upwards - fi - export IPHONEOS_DEPLOYMENT_TARGET=$PLATFORM - - # Build the requested iOS binaries - if [ "$GO_VERSION" -lt 150 ]; then - echo "Go version too low, skipping ios..." - else - # Add the 'ios' tag to all builds, otherwise the std libs will fail - if [ "$FLAG_TAGS" != "" ]; then - IOSTAGS=(--tags "ios $FLAG_TAGS") - else - IOSTAGS=(--tags ios) - fi - mkdir -p /build-ios-fw - - # Strip symbol table below Go 1.6 to prevent DWARF issues - LDSTRIP="" - if [ "$GO_VERSION" -lt 160 ]; then - LDSTRIP="-s" - fi - CGO_STATUS_IM="-D IOS_DEPLOYMENT" - # Cross compile to all available iOS and simulator platforms - if [ -d "$IOS_NDK_ARM_7" ] && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm-7" ] || [ $XGOARCH == "framework" ]); then - echo "Bootstrapping ios-$PLATFORM/arm-7..." - export PATH=$IOS_NDK_ARM_7/bin:$PATH - GOOS=darwin GOARCH=arm GOARM=7 CGO_ENABLED=1 CC=arm-apple-darwin11-clang go install --tags ios std - - echo "Compiling for ios-$PLATFORM/arm-7..." - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ HOST=arm-apple-darwin11 PREFIX=/usr/local $BUILD_DEPS /deps ${DEPS_ARGS[@]} - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=arm GOARM=7 CGO_ENABLED=1 go get $V $X "${IOSTAGS[@]}" --ldflags="$V $LD" -d ./$PACK - if [ $XGOARCH == "." ] || [ $XGOARCH == "arm-7" ]; then - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=arm GOARM=7 CGO_ENABLED=1 go build $V $X "${IOSTAGS[@]}" --ldflags="$LDSTRIP $V $LD" $BM -o "/build/$NAME-ios-$PLATFORM-armv7`extension darwin`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "framework" ]; then - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=arm GOARM=7 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${IOSTAGS[@]}" --ldflags="$V $LD" --buildmode=c-archive -o "/build-ios-fw/$NAME-ios-$PLATFORM-armv7.a" ./$PACK - fi - echo "Cleaning up Go runtime for ios-$PLATFORM/arm-7..." - rm -rf /usr/local/go/pkg/darwin_arm - fi - if [ -d "$IOS_NDK_ARM64" ] && ([ $XGOARCH == "." ] || [ $XGOARCH == "arm64" ] || [ $XGOARCH == "framework" ]); then - echo "Bootstrapping ios-$PLATFORM/arm64..." - export PATH=$IOS_NDK_ARM64/bin:$PATH - GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 CC=arm-apple-darwin11-clang go install --tags ios std - - echo "Compiling for ios-$PLATFORM/arm64..." - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ HOST=arm-apple-darwin11 PREFIX=/usr/local $BUILD_DEPS /deps ${DEPS_ARGS[@]} - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go get $V $X "${IOSTAGS[@]}" --ldflags="$V $LD" -d ./$PACK - if [ $XGOARCH == "." ] || [ $XGOARCH == "arm64" ]; then - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${IOSTAGS[@]}" --ldflags="$LDSTRIP $V $LD" $BM -o "/build/$NAME-ios-$PLATFORM-arm64`extension darwin`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "framework" ]; then - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${IOSTAGS[@]}" --ldflags="$V $LD" --buildmode=c-archive -o "/build-ios-fw/$NAME-ios-$PLATFORM-arm64.a" ./$PACK - fi - echo "Cleaning up Go runtime for ios-$PLATFORM/arm64..." - rm -rf /usr/local/go/pkg/darwin_arm64 - fi - if [ -d "$IOS_SIM_NDK_AMD64" ] && ([ $XGOARCH == "." ] || [ $XGOARCH == "amd64" ] || [ $XGOARCH == "framework" ]); then - echo "Bootstrapping ios-$PLATFORM/amd64..." - export PATH=$IOS_SIM_NDK_AMD64/bin:$PATH - mv /usr/local/go/pkg/darwin_amd64 /usr/local/go/pkg/darwin_amd64_bak - GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 CC=arm-apple-darwin11-clang go install --tags ios std - - echo "Compiling for ios-$PLATFORM/amd64..." - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ HOST=arm-apple-darwin11 PREFIX=/usr/local $BUILD_DEPS /deps ${DEPS_ARGS[@]} - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get $V $X "${IOSTAGS[@]}" --ldflags="$V $LD" -d ./$PACK - if [ $XGOARCH == "." ] || [ $XGOARCH == "amd64" ]; then - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${IOSTAGS[@]}" --ldflags="$LDSTRIP $V $LD" $BM -o "/build/$NAME-ios-$PLATFORM-x86_64`extension darwin`" ./$PACK - fi - if [ $XGOARCH == "." ] || [ $XGOARCH == "framework" ]; then - CC=arm-apple-darwin11-clang CXX=arm-apple-darwin11-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CGO_STATUS_IM" go build $V $X "${IOSTAGS[@]}" --ldflags="$V $LD" --buildmode=c-archive -o "/build-ios-fw/$NAME-ios-$PLATFORM-x86_64.a" ./$PACK - fi - echo "Cleaning up Go runtime for ios-$PLATFORM/amd64..." - rm -rf /usr/local/go/pkg/darwin_amd64 - mv /usr/local/go/pkg/darwin_amd64_bak /usr/local/go/pkg/darwin_amd64 - fi - unset CGO_STATUS_IM - # Assemble the iOS framework from the built binaries - if [ $XGOARCH == "." ] || [ $XGOARCH == "framework" ]; then - title=${NAME^} - framework=/build/$NAME-ios-$PLATFORM-framework/$title.framework - - rm -rf $framework - mkdir -p $framework/Versions/A - (cd $framework/Versions && ln -nsf A Current) - - arches=() - for lib in `ls /build-ios-fw | grep -e '\.a$'`; do - arches+=("-arch" "`echo ${lib##*-} | cut -d '.' -f 1`" "/build-ios-fw/$lib") - done - arm-apple-darwin11-lipo -create "${arches[@]}" -o $framework/Versions/A/$title - arm-apple-darwin11-ranlib $framework/Versions/A/$title - (cd $framework && ln -nsf Versions/A/$title $title) - - mkdir -p $framework/Versions/A/Headers - for header in `ls /build-ios-fw | grep -e '\.h$'`; do - cp -f /build-ios-fw/$header $framework/Versions/A/Headers/$title.h - done - (cd $framework && ln -nsf Versions/A/Headers Headers) - - echo "Patching Statusgo.h to work correctly on any arch (32bit, 64bit)" - (cd $framework && perl -i -p0e 's/(\/\*\n\s*static assertion[\s\n\r\S+]+)(typedef char _check_for[^;]+;)/\1#ifdef __LP64__\ntypedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64\/8 ? 1:-1];\n#else\ntypedef char _check_for_32_bit_pointer_matching_GoInt[sizeof(void*)==32\/8 ? 1:-1];\n#endif/igm' Headers/Statusgo.h) - - mkdir -p $framework/Versions/A/Resources - echo -e "\n\n\n\n\n" > $framework/Versions/A/Resources/Info.plist - (cd $framework && ln -nsf Versions/A/Resources Resources) - - mkdir -p $framework/Versions/A/Modules - echo -e "framework module \"$title\" {\n header \"$title.h\"\n export *\n}" > $framework/Versions/A/Modules/module.modulemap - (cd $framework && ln -nsf Versions/A/Modules Modules) - - # Fix up permissions on bundle file - chown $UID:$GID /build/$NAME-ios-$PLATFORM-framework - chmod 777 -R /build/$NAME-ios-$PLATFORM-framework - fi - rm -rf /build-ios-fw - fi - # Remove any automatically injected deployment target vars - unset IPHONEOS_DEPLOYMENT_TARGET - fi -done - -# Clean up any leftovers for subsequent build invocations -echo "Cleaning up build environment..." -rm -rf /deps - -for dir in `ls /usr/local`; do - keep=0 - - # Check against original folder contents - for old in $USR_LOCAL_CONTENTS; do - if [ "$old" == "$dir" ]; then - keep=1 - fi - done - # Delete anything freshly generated - if [ "$keep" == "0" ]; then - rm -rf "/usr/local/$dir" - fi -done diff --git a/_assets/build/xgo/ios-simulator/Dockerfile b/_assets/build/xgo/ios-simulator/Dockerfile deleted file mode 100644 index 0b25f2bfe44..00000000000 --- a/_assets/build/xgo/ios-simulator/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM statusteam/xgo:1.10.x - -# ios installer doesn't work with the default tar binary in Docker -# (divan: I stole this solution here: https://github.com/coreos/bugs/issues/1095#issuecomment-350574389) -# https://github.com/moby/moby/issues/19647 -RUN apt-get update \ - && apt-get install -y --no-install-recommends bsdtar \ - && apt-get clean \ - && cp $(which tar) $(which tar)~ \ - && ln -sf $(which bsdtar) $(which tar) - -ADD update_ios.sh /update_ios.sh -ENV UPDATE_IOS /update_ios.sh -RUN chmod +x $UPDATE_IOS - -RUN \ - IOS_SDK_PATH=https://s3.amazonaws.com/farazdagi/status-im/iPhoneSimulator9.3.sdk.tar.gz && \ - $FETCH $IOS_SDK_PATH 460423bf776e651a84c6e1bc035dbce23f93e685 && \ - $UPDATE_IOS /iPhoneSimulator9.3.sdk.tar.gz && \ - rm -rf /iPhoneSimulator9.3.sdk.tar.gz diff --git a/_assets/build/xgo/ios-simulator/update_ios.sh b/_assets/build/xgo/ios-simulator/update_ios.sh deleted file mode 100644 index 896d8355bcf..00000000000 --- a/_assets/build/xgo/ios-simulator/update_ios.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# Contains a simple tool that updates some of the iOS toolchains with the SDKs -# explicitly provided. The goal is to allow using your own up to date SDKs or -# the simulator one not supported out of the box. -# -# Usage: update_ios.sh /.sdk.tar. -set -e - -# Figure out the base name of the SDK -sdk=`basename $1` -sdk=${sdk%.*} -sdk=${sdk%.*} - -# Define a small extraction utility to -function extract { - case $1 in - *.tar.xz) - xz -dc $1 | tar xf - - ;; - *.tar.gz) - gunzip -dc $1 | tar xf - - ;; - *.tar.bz2) - bzip2 -dc $1 | tar xf - - ;; - esac -} - -# Extract the SDK, patch it, clean it up and prep for bootstrapping -extract $1 - -if [[ "`basename $1`" =~ ^iPhoneSimulator ]]; then - echo "Patching iOS simulator SDK with missing libraries..." - ln -s $OSX_NDK_X86/SDK/$OSX_SDK/usr/lib/system/libsystem_kernel.dylib $sdk/usr/lib/system/libsystem_kernel.dylib - ln -s $OSX_NDK_X86/SDK/$OSX_SDK/usr/lib/system/libsystem_platform.dylib $sdk/usr/lib/system/libsystem_platform.dylib - ln -s $OSX_NDK_X86/SDK/$OSX_SDK/usr/lib/system/libsystem_pthread.dylib $sdk/usr/lib/system/libsystem_pthread.dylib - ln -s $OSX_NDK_X86/SDK/$OSX_SDK/usr/lib/system/libsystem_kernel.tbd $sdk/usr/lib/system/libsystem_kernel.tbd - ln -s $OSX_NDK_X86/SDK/$OSX_SDK/usr/lib/system/libsystem_platform.tbd $sdk/usr/lib/system/libsystem_platform.tbd - ln -s $OSX_NDK_X86/SDK/$OSX_SDK/usr/lib/system/libsystem_pthread.tbd $sdk/usr/lib/system/libsystem_pthread.tbd -fi - -tar -czf /tmp/$sdk.tar.gz $sdk -rm -rf $sdk - -# Pull the iOS cross compiler tool and build the toolchain -git clone -n https://github.com/tpoechtrager/cctools-port.git -cd cctools-port -git reset --hard adf616eee9d41f4961c3a83ba275249ffcb32d33 -cd .. - -if [[ "`basename $1`" =~ ^iPhoneSimulator ]]; then - rm -rf $IOS_SIM_NDK_AMD64 - /cctools-port/usage_examples/ios_toolchain/build.sh /tmp/$sdk.tar.gz x86_64 - mv /cctools-port/usage_examples/ios_toolchain/target $IOS_SIM_NDK_AMD64 -else - rm -rf $IOS_NDK_ARM_7 $IOS_NDK_ARM64 - /cctools-port/usage_examples/ios_toolchain/build.sh /tmp/$sdk.tar.gz armv7 - mv /cctools-port/usage_examples/ios_toolchain/target $IOS_NDK_ARM_7 - /cctools-port/usage_examples/ios_toolchain/build.sh /tmp/$sdk.tar.gz arm64 - mv /cctools-port/usage_examples/ios_toolchain/target $IOS_NDK_ARM64 -fi - -rm -rf /cctools-port diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 741cc4e527f..2c4fa9e5424 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -49,7 +49,7 @@ pipeline { } } } stage('Archive') { steps { sh """ - mv ${env.STATUS_PATH}/build/bin/statusgo-android-16.aar \ + mv ${env.STATUS_PATH}/build/bin/statusgo.aar \ ${env.WORKSPACE}/status-go-android-${lib.suffix()}.aar """ archiveArtifacts("status-go-android-${lib.suffix()}.aar") @@ -62,8 +62,8 @@ pipeline { stage('iOS') { stages { stage('Compile') { steps { dir(env.STATUS_PATH) { - sh 'make statusgo-ios-simulator' - dir('build/bin/statusgo-ios-9.3-framework') { + sh 'make statusgo-ios' + dir('build/bin/Statusgo.framework') { sh 'zip -r status-go-ios.zip Statusgo.framework' } } } } diff --git a/_assets/patches/geth-xgo/0001-fix-duktapev3-missing-SIZE_MAX-def.patch b/_assets/patches/geth-xgo/0001-fix-duktapev3-missing-SIZE_MAX-def.patch deleted file mode 100644 index 4652dc16299..00000000000 --- a/_assets/patches/geth-xgo/0001-fix-duktapev3-missing-SIZE_MAX-def.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/vendor/gopkg.in/olebedev/go-duktape.v3/duk_minimal_printf.c b/vendor/gopkg.in/olebedev/go-duktape.v3/duk_minimal_printf.c -index e4b6e43a..baed990d 100755 ---- a/vendor/gopkg.in/olebedev/go-duktape.v3/duk_minimal_printf.c -+++ b/vendor/gopkg.in/olebedev/go-duktape.v3/duk_minimal_printf.c -@@ -278,6 +278,7 @@ int duk_minimal_snprintf(char *str, size_t size, const char *format, ...) { - } - - /* Minimal sprintf() entry point. */ -+#if 0 - int duk_minimal_sprintf(char *str, const char *format, ...) { - va_list ap; - int ret; -@@ -288,6 +289,7 @@ int duk_minimal_sprintf(char *str, const char *format, ...) { - - return ret; - } -+#endif - - /* Minimal sscanf() entry point. */ - int duk_minimal_sscanf(const char *str, const char *format, ...) { diff --git a/_assets/patches/geth-xgo/0002-remove-dashboard-collectData.patch b/_assets/patches/geth-xgo/0002-remove-dashboard-collectData.patch deleted file mode 100644 index a18bcaa8f50..00000000000 --- a/_assets/patches/geth-xgo/0002-remove-dashboard-collectData.patch +++ /dev/null @@ -1,153 +0,0 @@ -diff --git a/vendor/github.com/ethereum/go-ethereum/dashboard/dashboard.go b/vendor/github.com/ethereum/go-ethereum/dashboard/dashboard.go -index 3ba92ac7..d89a1e94 100644 ---- a/vendor/github.com/ethereum/go-ethereum/dashboard/dashboard.go -+++ b/vendor/github.com/ethereum/go-ethereum/dashboard/dashboard.go -@@ -27,20 +27,17 @@ import ( - "fmt" - "net" - "net/http" -- "runtime" - "sync" - "sync/atomic" - "time" - - "io" - -- "github.com/elastic/gosigar" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rpc" -- "github.com/mohae/deepcopy" - "golang.org/x/net/websocket" - ) - -@@ -235,7 +232,7 @@ func (db *Dashboard) apiHandler(conn *websocket.Conn) { - - db.lock.Lock() - // Send the past data. -- client.msg <- deepcopy.Copy(db.history).(*Message) -+ client.msg <- db.history - // Start tracking the connection and drop at connection loss. - db.conns[id] = client - db.lock.Unlock() -@@ -275,118 +272,6 @@ func meterCollector(name string) func() int64 { - // collectData collects the required data to plot on the dashboard. - func (db *Dashboard) collectData() { - defer db.wg.Done() -- -- systemCPUUsage := gosigar.Cpu{} -- systemCPUUsage.Get() -- var ( -- mem runtime.MemStats -- -- collectNetworkIngress = meterCollector("p2p/InboundTraffic") -- collectNetworkEgress = meterCollector("p2p/OutboundTraffic") -- collectDiskRead = meterCollector("eth/db/chaindata/disk/read") -- collectDiskWrite = meterCollector("eth/db/chaindata/disk/write") -- -- prevNetworkIngress = collectNetworkIngress() -- prevNetworkEgress = collectNetworkEgress() -- prevProcessCPUTime = getProcessCPUTime() -- prevSystemCPUUsage = systemCPUUsage -- prevDiskRead = collectDiskRead() -- prevDiskWrite = collectDiskWrite() -- -- frequency = float64(db.config.Refresh / time.Second) -- numCPU = float64(runtime.NumCPU()) -- ) -- -- for { -- select { -- case errc := <-db.quit: -- errc <- nil -- return -- case <-time.After(db.config.Refresh): -- systemCPUUsage.Get() -- var ( -- curNetworkIngress = collectNetworkIngress() -- curNetworkEgress = collectNetworkEgress() -- curProcessCPUTime = getProcessCPUTime() -- curSystemCPUUsage = systemCPUUsage -- curDiskRead = collectDiskRead() -- curDiskWrite = collectDiskWrite() -- -- deltaNetworkIngress = float64(curNetworkIngress - prevNetworkIngress) -- deltaNetworkEgress = float64(curNetworkEgress - prevNetworkEgress) -- deltaProcessCPUTime = curProcessCPUTime - prevProcessCPUTime -- deltaSystemCPUUsage = curSystemCPUUsage.Delta(prevSystemCPUUsage) -- deltaDiskRead = curDiskRead - prevDiskRead -- deltaDiskWrite = curDiskWrite - prevDiskWrite -- ) -- prevNetworkIngress = curNetworkIngress -- prevNetworkEgress = curNetworkEgress -- prevProcessCPUTime = curProcessCPUTime -- prevSystemCPUUsage = curSystemCPUUsage -- prevDiskRead = curDiskRead -- prevDiskWrite = curDiskWrite -- -- now := time.Now() -- -- runtime.ReadMemStats(&mem) -- activeMemory := &ChartEntry{ -- Time: now, -- Value: float64(mem.Alloc) / frequency, -- } -- virtualMemory := &ChartEntry{ -- Time: now, -- Value: float64(mem.Sys) / frequency, -- } -- networkIngress := &ChartEntry{ -- Time: now, -- Value: deltaNetworkIngress / frequency, -- } -- networkEgress := &ChartEntry{ -- Time: now, -- Value: deltaNetworkEgress / frequency, -- } -- processCPU := &ChartEntry{ -- Time: now, -- Value: deltaProcessCPUTime / frequency / numCPU * 100, -- } -- systemCPU := &ChartEntry{ -- Time: now, -- Value: float64(deltaSystemCPUUsage.Sys+deltaSystemCPUUsage.User) / frequency / numCPU, -- } -- diskRead := &ChartEntry{ -- Time: now, -- Value: float64(deltaDiskRead) / frequency, -- } -- diskWrite := &ChartEntry{ -- Time: now, -- Value: float64(deltaDiskWrite) / frequency, -- } -- sys := db.history.System -- db.lock.Lock() -- sys.ActiveMemory = append(sys.ActiveMemory[1:], activeMemory) -- sys.VirtualMemory = append(sys.VirtualMemory[1:], virtualMemory) -- sys.NetworkIngress = append(sys.NetworkIngress[1:], networkIngress) -- sys.NetworkEgress = append(sys.NetworkEgress[1:], networkEgress) -- sys.ProcessCPU = append(sys.ProcessCPU[1:], processCPU) -- sys.SystemCPU = append(sys.SystemCPU[1:], systemCPU) -- sys.DiskRead = append(sys.DiskRead[1:], diskRead) -- sys.DiskWrite = append(sys.DiskWrite[1:], diskWrite) -- db.lock.Unlock() -- -- db.sendToAll(&Message{ -- System: &SystemMessage{ -- ActiveMemory: ChartEntries{activeMemory}, -- VirtualMemory: ChartEntries{virtualMemory}, -- NetworkIngress: ChartEntries{networkIngress}, -- NetworkEgress: ChartEntries{networkEgress}, -- ProcessCPU: ChartEntries{processCPU}, -- SystemCPU: ChartEntries{systemCPU}, -- DiskRead: ChartEntries{diskRead}, -- DiskWrite: ChartEntries{diskWrite}, -- }, -- }) -- } -- } - } - - // sendToAll sends the given message to the active dashboards. diff --git a/_assets/patches/geth-xgo/README.md b/_assets/patches/geth-xgo/README.md deleted file mode 100644 index f8cf045ee3f..00000000000 --- a/_assets/patches/geth-xgo/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Status Patches for geth (go-ethereum) cross-compiled in Xgo ---- - -Status-go uses [go-ethereum](https://github.com/ethereum/go-ethereum) (**upstream**) as its dependency. When cross-compiling with Xgo, some headers or definitions are not available within the Xgo environment. In such a situation, we temporarily patch the sources before kicking the build in Xgo and revert them afterwards (this is taken care by the respective Makefile targets). - -We try to minimize number and amount of changes in those patches as much as possible, and whereas possible, to contribute changes into the upstream. - -# Creating patches - -Instructions for creating a patch from the command line: - -1. Enter the command line at the go-ethereum dependency root in vendor folder. -1. Create the patch: - 1. If you already have a commit that represents the change, find its SHA1 (e.g. `$COMMIT_SHA1`) and do `git diff $COMMIT_SHA1 > file.patch` - 1. If the files are staged, do `git diff --cached > file.patch` - -# Patches - -- [`0001-fix-duktapev3-missing-SIZE_MAX-def.patch`](./0001-fix-duktapev3-missing-SIZE_MAX-def.patch) — Adds patch to geth 1.8.0 dependency duktapev3, to address issue where SIZE_MAX is not defined in xgo for Android -- [`0002-remove-dashboard-collectData.patch`](./0002-remove-dashboard-collectData.patch) — Deletes the body of `collectData` in the `dashboard` package, since it will import the `gosigar` package which in turn includes a header (`libproc.h`) which is missing in the iOS environment in Xgo. From 6e217ef9f656884d791774bd00177279502998d4 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 11:36:32 +0100 Subject: [PATCH 08/61] prepare Jenkinsfile --- .gitignore | 1 - Makefile | 7 +-- _assets/ci/Jenkinsfile | 116 ++++++++++++++++++++++++++++------------- 3 files changed, 83 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index b2123dae795..176f1b7e06a 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,6 @@ tags # used by the Makefile /build/_workspace/ /build/bin/ -/vendor/github.com/status-im/xgo # travis profile.tmp diff --git a/Makefile b/Makefile index f83920f457f..5e3803b1ce9 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ docker-image: ##@docker Build docker image (use DOCKER_IMAGE_NAME to set the ima -t $(DOCKER_IMAGE_NAME):latest bootnode-image: - @echo "Building docker image for bootnode..." + @echo "Building docker image for bootnode..."` docker build --file _assets/build/Dockerfile-bootnode . \ --build-arg "build_tags=$(BUILD_TAGS)" \ --build-arg "build_flags=$(BUILD_FLAGS)" \ @@ -189,8 +189,9 @@ setup: dep-install lint-install mock-install gomobile-install ##@other Prepare p prepare-release: clean-release mkdir -p $(RELEASE_DIRECTORY) - mv build/bin/statusgo-android-16.aar $(RELEASE_DIRECTORY)/status-go-android.aar - mv build/bin/statusgo-ios-9.3-framework/status-go-ios.zip $(RELEASE_DIRECTORY)/status-go-ios.zip + mv build/bin/statusgo.aar $(RELEASE_DIRECTORY)/status-go-android.aar + zip -r build/bin/Statusgo.framework.zip build/bin/Statusgo.framework + mv build/bin/Statusgo.framework.zip $(RELEASE_DIRECTORY)/status-go-ios.zip ${MAKE} clean zip -r $(RELEASE_DIRECTORY)/status-go-desktop.zip . -x *.git* diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 2c4fa9e5424..6b0f429c19b 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent { label 'linux' } + agent none parameters { booleanParam( @@ -27,22 +27,27 @@ pipeline { } stages { - stage('Prep') { - steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") - } } - } - stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup' - } } } - stage('Lint') { steps { dir(env.STATUS_PATH) { - sh 'make ci' - } } } - stage('Build') { - parallel { - stage('Android') { + parallel { + stage('Android') { + agent { label 'linux' } + + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + } } + } + + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup' + } } } + + stage('Lint') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } + + stage('Build') { stages { stage('Compile') { steps { dir(env.STATUS_PATH) { sh 'make statusgo-android' @@ -59,17 +64,53 @@ pipeline { } } } } } - stage('iOS') { + + stage('Release') { when { expression { params.GITHUB_RELEASE == true } } + steps { + sh 'make prepare-release' + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'status-im-auto', + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ]]) { + sh "yes | make release release_branch=${gitBranch}" + } + sh 'make clean-release' + } + } + } // Android + + stage('iOS') { + agent { label 'macos' } + + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + } } + } + + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup' + } } } + + stage('Lint') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } + + stage('Build') { stages { stage('Compile') { steps { dir(env.STATUS_PATH) { sh 'make statusgo-ios' - dir('build/bin/Statusgo.framework') { - sh 'zip -r status-go-ios.zip Statusgo.framework' + dir('build/bin') { + sh 'zip -r Statusgo.framework.zip Statusgo.framework' } } } } stage('Archive') { steps { sh """ - mv ${env.STATUS_PATH}/build/bin/statusgo-ios-9.3-framework/status-go-ios.zip \ + mv ${env.STATUS_PATH}/build/bin/Statusgo.framework.zip \ ${env.WORKSPACE}/status-go-ios-${lib.suffix()}.zip """ archiveArtifacts("status-go-ios-${lib.suffix()}.zip") @@ -79,21 +120,22 @@ pipeline { } } } } } - } - } - stage('Release') { when { expression { params.GITHUB_RELEASE == true } } - steps { - sh 'make prepare-release' - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh "yes | make release release_branch=${gitBranch}" + + stage('Release') { when { expression { params.GITHUB_RELEASE == true } } + steps { + sh 'make prepare-release' + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'status-im-auto', + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ]]) { + sh "yes | make release release_branch=${gitBranch}" + } + sh 'make clean-release' + } } - sh 'make clean-release' - } - } - } + } // iOS + } // parallel + } // stages } From 43d965eff94d25e3e914794c8aeb9769b2df19af Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 11:40:52 +0100 Subject: [PATCH 09/61] fix Jenkinsfile --- _assets/ci/Jenkinsfile | 188 +++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 93 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 6b0f429c19b..612b7c15a32 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -27,115 +27,117 @@ pipeline { } stages { - parallel { - stage('Android') { - agent { label 'linux' } + stage('Parallel builds') { + parallel { + stage('Android') { + agent { label 'linux' } - stage('Prep') { - steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") - } } - } + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + } } + } - stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup' - } } } + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup' + } } } - stage('Lint') { steps { dir(env.STATUS_PATH) { - sh 'make ci' - } } } + stage('Lint') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } - stage('Build') { - stages { - stage('Compile') { steps { dir(env.STATUS_PATH) { - sh 'make statusgo-android' - } } } - stage('Archive') { steps { - sh """ - mv ${env.STATUS_PATH}/build/bin/statusgo.aar \ - ${env.WORKSPACE}/status-go-android-${lib.suffix()}.aar - """ - archiveArtifacts("status-go-android-${lib.suffix()}.aar") - } } - stage('Upload') { steps { script { - lib.uploadArtifact("status-go-android-${lib.suffix()}.aar") - } } } + stage('Build') { + stages { + stage('Compile') { steps { dir(env.STATUS_PATH) { + sh 'make statusgo-android' + } } } + stage('Archive') { steps { + sh """ + mv ${env.STATUS_PATH}/build/bin/statusgo.aar \ + ${env.WORKSPACE}/status-go-android-${lib.suffix()}.aar + """ + archiveArtifacts("status-go-android-${lib.suffix()}.aar") + } } + stage('Upload') { steps { script { + lib.uploadArtifact("status-go-android-${lib.suffix()}.aar") + } } } + } } - } - stage('Release') { when { expression { params.GITHUB_RELEASE == true } } - steps { - sh 'make prepare-release' - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh "yes | make release release_branch=${gitBranch}" + stage('Release') { when { expression { params.GITHUB_RELEASE == true } } + steps { + sh 'make prepare-release' + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'status-im-auto', + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ]]) { + sh "yes | make release release_branch=${gitBranch}" + } + sh 'make clean-release' } - sh 'make clean-release' } - } - } // Android + } // Android - stage('iOS') { - agent { label 'macos' } + stage('iOS') { + agent { label 'macos' } - stage('Prep') { - steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") - } } - } + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + } } + } - stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup' - } } } + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup' + } } } - stage('Lint') { steps { dir(env.STATUS_PATH) { - sh 'make ci' - } } } + stage('Lint') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } - stage('Build') { - stages { - stage('Compile') { steps { dir(env.STATUS_PATH) { - sh 'make statusgo-ios' - dir('build/bin') { - sh 'zip -r Statusgo.framework.zip Statusgo.framework' - } - } } } - stage('Archive') { steps { - sh """ - mv ${env.STATUS_PATH}/build/bin/Statusgo.framework.zip \ - ${env.WORKSPACE}/status-go-ios-${lib.suffix()}.zip - """ - archiveArtifacts("status-go-ios-${lib.suffix()}.zip") - } } - stage('Upload') { steps { script { - lib.uploadArtifact("status-go-ios-${lib.suffix()}.zip") - } } } + stage('Build') { + stages { + stage('Compile') { steps { dir(env.STATUS_PATH) { + sh 'make statusgo-ios' + dir('build/bin') { + sh 'zip -r Statusgo.framework.zip Statusgo.framework' + } + } } } + stage('Archive') { steps { + sh """ + mv ${env.STATUS_PATH}/build/bin/Statusgo.framework.zip \ + ${env.WORKSPACE}/status-go-ios-${lib.suffix()}.zip + """ + archiveArtifacts("status-go-ios-${lib.suffix()}.zip") + } } + stage('Upload') { steps { script { + lib.uploadArtifact("status-go-ios-${lib.suffix()}.zip") + } } } + } } - } - stage('Release') { when { expression { params.GITHUB_RELEASE == true } } - steps { - sh 'make prepare-release' - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh "yes | make release release_branch=${gitBranch}" + stage('Release') { when { expression { params.GITHUB_RELEASE == true } } + steps { + sh 'make prepare-release' + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'status-im-auto', + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ]]) { + sh "yes | make release release_branch=${gitBranch}" + } + sh 'make clean-release' } - sh 'make clean-release' } - } - } // iOS - } // parallel + } // iOS + } // parallel + } // stage } // stages } From 6c68d395f74ffab6e32186e3848a76f8f5588fd9 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 11:49:19 +0100 Subject: [PATCH 10/61] add steps --- _assets/ci/Jenkinsfile | 186 +++++++++++++++++++++-------------------- 1 file changed, 94 insertions(+), 92 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 612b7c15a32..c878dd5bef1 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -27,116 +27,118 @@ pipeline { } stages { - stage('Parallel builds') { + stage('Parallel Builds') { parallel { - stage('Android') { - agent { label 'linux' } + steps { + stage('Android') { + agent { label 'linux' } - stage('Prep') { - steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") - } } - } + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + } } + } - stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup' - } } } + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup' + } } } - stage('Lint') { steps { dir(env.STATUS_PATH) { - sh 'make ci' - } } } + stage('Lint') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } - stage('Build') { - stages { - stage('Compile') { steps { dir(env.STATUS_PATH) { - sh 'make statusgo-android' - } } } - stage('Archive') { steps { - sh """ - mv ${env.STATUS_PATH}/build/bin/statusgo.aar \ - ${env.WORKSPACE}/status-go-android-${lib.suffix()}.aar - """ - archiveArtifacts("status-go-android-${lib.suffix()}.aar") - } } - stage('Upload') { steps { script { - lib.uploadArtifact("status-go-android-${lib.suffix()}.aar") - } } } + stage('Build') { + stages { + stage('Compile') { steps { dir(env.STATUS_PATH) { + sh 'make statusgo-android' + } } } + stage('Archive') { steps { + sh """ + mv ${env.STATUS_PATH}/build/bin/statusgo.aar \ + ${env.WORKSPACE}/status-go-android-${lib.suffix()}.aar + """ + archiveArtifacts("status-go-android-${lib.suffix()}.aar") + } } + stage('Upload') { steps { script { + lib.uploadArtifact("status-go-android-${lib.suffix()}.aar") + } } } + } } - } - stage('Release') { when { expression { params.GITHUB_RELEASE == true } } - steps { - sh 'make prepare-release' - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh "yes | make release release_branch=${gitBranch}" + stage('Release') { when { expression { params.GITHUB_RELEASE == true } } + steps { + sh 'make prepare-release' + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'status-im-auto', + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ]]) { + sh "yes | make release release_branch=${gitBranch}" + } + sh 'make clean-release' } - sh 'make clean-release' } - } - } // Android + } // Android - stage('iOS') { - agent { label 'macos' } + stage('iOS') { + agent { label 'macos' } - stage('Prep') { - steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") - } } - } + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + } } + } - stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup' - } } } + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup' + } } } - stage('Lint') { steps { dir(env.STATUS_PATH) { - sh 'make ci' - } } } + stage('Lint') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } - stage('Build') { - stages { - stage('Compile') { steps { dir(env.STATUS_PATH) { - sh 'make statusgo-ios' - dir('build/bin') { - sh 'zip -r Statusgo.framework.zip Statusgo.framework' - } - } } } - stage('Archive') { steps { - sh """ - mv ${env.STATUS_PATH}/build/bin/Statusgo.framework.zip \ - ${env.WORKSPACE}/status-go-ios-${lib.suffix()}.zip - """ - archiveArtifacts("status-go-ios-${lib.suffix()}.zip") - } } - stage('Upload') { steps { script { - lib.uploadArtifact("status-go-ios-${lib.suffix()}.zip") - } } } + stage('Build') { + stages { + stage('Compile') { steps { dir(env.STATUS_PATH) { + sh 'make statusgo-ios' + dir('build/bin') { + sh 'zip -r Statusgo.framework.zip Statusgo.framework' + } + } } } + stage('Archive') { steps { + sh """ + mv ${env.STATUS_PATH}/build/bin/Statusgo.framework.zip \ + ${env.WORKSPACE}/status-go-ios-${lib.suffix()}.zip + """ + archiveArtifacts("status-go-ios-${lib.suffix()}.zip") + } } + stage('Upload') { steps { script { + lib.uploadArtifact("status-go-ios-${lib.suffix()}.zip") + } } } + } } - } - stage('Release') { when { expression { params.GITHUB_RELEASE == true } } - steps { - sh 'make prepare-release' - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh "yes | make release release_branch=${gitBranch}" + stage('Release') { when { expression { params.GITHUB_RELEASE == true } } + steps { + sh 'make prepare-release' + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'status-im-auto', + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ]]) { + sh "yes | make release release_branch=${gitBranch}" + } + sh 'make clean-release' } - sh 'make clean-release' } - } - } // iOS + } // iOS + } // steps } // parallel } // stage } // stages From c4063f073362bde156bdd11d516ed090427d7a00 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 12:07:23 +0100 Subject: [PATCH 11/61] try now --- _assets/ci/Jenkinsfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index c878dd5bef1..751b51d7325 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -27,12 +27,12 @@ pipeline { } stages { - stage('Parallel Builds') { + stage('Build') { parallel { - steps { - stage('Android') { - agent { label 'linux' } + stage('Android') { + agent { label 'linux' } + stages { stage('Prep') { steps { script { lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") @@ -65,7 +65,6 @@ pipeline { lib.uploadArtifact("status-go-android-${lib.suffix()}.aar") } } } } - } stage('Release') { when { expression { params.GITHUB_RELEASE == true } } steps { @@ -81,11 +80,13 @@ pipeline { sh 'make clean-release' } } - } // Android + } + } // Android - stage('iOS') { - agent { label 'macos' } + stage('iOS') { + agent { label 'macos' } + stages { stage('Prep') { steps { script { lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") @@ -137,8 +138,8 @@ pipeline { sh 'make clean-release' } } - } // iOS - } // steps + } + } // iOS } // parallel } // stage } // stages From 55a28cb04cbb22b3f10583467485369ca49df897 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 12:22:09 +0100 Subject: [PATCH 12/61] fix parentheses --- _assets/ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 751b51d7325..d313b359cf6 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -65,6 +65,7 @@ pipeline { lib.uploadArtifact("status-go-android-${lib.suffix()}.aar") } } } } + } stage('Release') { when { expression { params.GITHUB_RELEASE == true } } steps { From 9d76ebeb2fbefc6f01cf4afedf602364df5f3c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Mon, 26 Nov 2018 14:50:22 +0100 Subject: [PATCH 13/61] fix location of envrionment section --- _assets/ci/Jenkinsfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index d313b359cf6..0fd6fec4e4d 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -20,18 +20,18 @@ pipeline { )) } - environment { - STATUS_PATH = 'src/github.com/status-im/status-go' - GOPATH = "${env.WORKSPACE}" - PATH = "${env.PATH}:${env.GOPATH}/bin" - } - stages { stage('Build') { parallel { stage('Android') { agent { label 'linux' } + environment { + STATUS_PATH = 'src/github.com/status-im/status-go' + GOPATH = "${env.WORKSPACE}" + PATH = "${env.PATH}:${env.GOPATH}/bin" + } + stages { stage('Prep') { steps { script { @@ -87,6 +87,12 @@ pipeline { stage('iOS') { agent { label 'macos' } + environment { + STATUS_PATH = 'src/github.com/status-im/status-go' + GOPATH = "${env.WORKSPACE}" + PATH = "${env.PATH}:${env.GOPATH}/bin" + } + stages { stage('Prep') { steps { script { From 275d35d5edea376c7bd81c01d935e701e7c942bf Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 15:18:49 +0100 Subject: [PATCH 14/61] split setup --- Makefile | 6 +++++- _assets/ci/Jenkinsfile | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5e3803b1ce9..07cfd3c120d 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,11 @@ endif install-os-dependencies: _assets/scripts/install_deps.sh -setup: install-os-dependencies dep-install lint-install mock-install deploy-install gen-install update-fleet-config ##@other Prepare project for first build +setup-dev: setup-build install-os-dependencies gen-install update-fleet-config ##@other Prepare project for development + +setup-build: dep-install lint-install mock-install deploy-install ##@other Prepare project for build + +setup: setup-build setup-dev ##@other Prepare project for development and building generate: ##@other Regenerate assets and other auto-generated stuff go generate ./static ./static/migrations diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 0fd6fec4e4d..676267206a5 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -42,7 +42,7 @@ pipeline { } stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup' + sh 'make setup-build' } } } stage('Lint') { steps { dir(env.STATUS_PATH) { @@ -103,7 +103,7 @@ pipeline { } stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup' + sh 'make setup-build' } } } stage('Lint') { steps { dir(env.STATUS_PATH) { From 1c6a332eea06d7643c43794267bb518d85964a41 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 15:37:19 +0100 Subject: [PATCH 15/61] skip TestGetFilterLogs --- services/rpcfilters/api_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/rpcfilters/api_test.go b/services/rpcfilters/api_test.go index da18171c45e..c4ded952381 100644 --- a/services/rpcfilters/api_test.go +++ b/services/rpcfilters/api_test.go @@ -79,6 +79,8 @@ func TestGetFilterChangesResetsTimer(t *testing.T) { } func TestGetFilterLogs(t *testing.T) { + t.Skip("Skipping due to flakiness: https://github.com/status-im/status-go/issues/1281") + tracker := new(callTracker) api := &PublicAPI{ filters: make(map[rpc.ID]filter), From 7dc20aec97966be58c64fc54a513fc77e237feb7 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 15:45:14 +0100 Subject: [PATCH 16/61] disable USB detection --- services/shhext/service_test.go | 1 + t/devtests/devnode.go | 1 + 2 files changed, 2 insertions(+) diff --git a/services/shhext/service_test.go b/services/shhext/service_test.go index c808979598a..e23b900e14e 100644 --- a/services/shhext/service_test.go +++ b/services/shhext/service_test.go @@ -80,6 +80,7 @@ func (s *ShhExtSuite) SetupTest() { MaxPeers: 1, ListenAddr: ":0", }, + NoUSB: true, } stack, err := node.New(cfg) s.NoError(err) diff --git a/t/devtests/devnode.go b/t/devtests/devnode.go index fcee535f25b..e1c9d0dfac0 100644 --- a/t/devtests/devnode.go +++ b/t/devtests/devnode.go @@ -30,6 +30,7 @@ func NewDevNode(faucet common.Address) (*node.Node, error) { cfg.IPCPath = ipc.Name() cfg.HTTPModules = []string{"eth"} cfg.DataDir = "" + cfg.NoUSB = true cfg.P2P.MaxPeers = 0 cfg.P2P.ListenAddr = ":0" cfg.P2P.NoDiscovery = true From 4cb9885419f45b9d129c7d0f1570ca42f1ee4075 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 15:54:36 +0100 Subject: [PATCH 17/61] run gomobile-install in setup-build --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 07cfd3c120d..8b59dba6f31 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,7 @@ install-os-dependencies: setup-dev: setup-build install-os-dependencies gen-install update-fleet-config ##@other Prepare project for development -setup-build: dep-install lint-install mock-install deploy-install ##@other Prepare project for build +setup-build: dep-install lint-install mock-install deploy-install gomobile-install ##@other Prepare project for build setup: setup-build setup-dev ##@other Prepare project for development and building @@ -189,8 +189,6 @@ generate: ##@other Regenerate assets and other auto-generated stuff go generate ./static ./static/migrations $(shell cd ./services/shhext/chat && exec protoc --go_out=. ./*.proto) -setup: dep-install lint-install mock-install gomobile-install ##@other Prepare project for first build - prepare-release: clean-release mkdir -p $(RELEASE_DIRECTORY) mv build/bin/statusgo.aar $(RELEASE_DIRECTORY)/status-go-android.aar @@ -216,6 +214,7 @@ release: gomobile-install: go get -u golang.org/x/mobile/cmd/gomobile + gomobile init deploy-install: go get -u github.com/c4milo/github-release From 719711e421b4943c53d03f7796d1b5fd7d08e52f Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 16:04:09 +0100 Subject: [PATCH 18/61] add NoUSB in all places --- metrics/node/metrics_test.go | 1 + metrics/node/subscribe_test.go | 4 ++-- node/status_node_test.go | 2 ++ services/shhext/service_test.go | 3 +++ t/benchmarks/utils_test.go | 1 + 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/metrics/node/metrics_test.go b/metrics/node/metrics_test.go index 2ece2bc8ef1..00383b267e0 100644 --- a/metrics/node/metrics_test.go +++ b/metrics/node/metrics_test.go @@ -15,6 +15,7 @@ func TestUpdateNodeMetricsPeersCounter(t *testing.T) { P2P: p2p.Config{ MaxPeers: 10, }, + NoUSB: true, }) require.NoError(t, err) require.NoError(t, n.Start()) diff --git a/metrics/node/subscribe_test.go b/metrics/node/subscribe_test.go index f2c3ec634be..d5fd27dc170 100644 --- a/metrics/node/subscribe_test.go +++ b/metrics/node/subscribe_test.go @@ -9,13 +9,13 @@ import ( ) func TestSubscribeServerEventsWithoutServer(t *testing.T) { - gethNode, err := node.New(&node.Config{}) + gethNode, err := node.New(&node.Config{NoUSB: true}) require.NoError(t, err) require.EqualError(t, SubscribeServerEvents(context.TODO(), gethNode), "server is unavailable") } func TestSubscribeServerEvents(t *testing.T) { - gethNode, err := node.New(&node.Config{}) + gethNode, err := node.New(&node.Config{NoUSB: true}) require.NoError(t, err) err = gethNode.Start() require.NoError(t, err) diff --git a/node/status_node_test.go b/node/status_node_test.go index 8d2fc40528d..9542620574c 100644 --- a/node/status_node_test.go +++ b/node/status_node_test.go @@ -165,6 +165,7 @@ func TestStatusNodeAddPeer(t *testing.T) { NoDiscovery: true, ListenAddr: ":0", }, + NoUSB: true, }) require.NoError(t, err) require.NoError(t, peer.Start()) @@ -200,6 +201,7 @@ func TestStatusNodeReconnectStaticPeers(t *testing.T) { NoDiscovery: true, ListenAddr: ":0", }, + NoUSB: true, }) require.NoError(t, err) require.NoError(t, peer.Start()) diff --git a/services/shhext/service_test.go b/services/shhext/service_test.go index e23b900e14e..7a7cb765a28 100644 --- a/services/shhext/service_test.go +++ b/services/shhext/service_test.go @@ -165,6 +165,7 @@ func (s *ShhExtSuite) TestRequestMessagesErrors() { MaxPeers: math.MaxInt32, NoDiscovery: true, }, + NoUSB: true, }) // in-memory node as no data dir s.NoError(err) err = aNode.Register(func(*node.ServiceContext) (node.Service, error) { @@ -233,6 +234,7 @@ func (s *ShhExtSuite) TestRequestMessagesSuccess() { MaxPeers: math.MaxInt32, NoDiscovery: true, }, + NoUSB: true, }) // in-memory node as no data dir s.NoError(err) err = aNode.Register(func(*node.ServiceContext) (node.Service, error) { return shh, nil }) @@ -260,6 +262,7 @@ func (s *ShhExtSuite) TestRequestMessagesSuccess() { NoDiscovery: true, ListenAddr: ":0", }, + NoUSB: true, }) // in-memory node as no data dir s.NoError(err) err = mailNode.Register(func(*node.ServiceContext) (node.Service, error) { diff --git a/t/benchmarks/utils_test.go b/t/benchmarks/utils_test.go index 1a0af3c5b00..41437445032 100644 --- a/t/benchmarks/utils_test.go +++ b/t/benchmarks/utils_test.go @@ -30,6 +30,7 @@ func createNode() (*node.Node, error) { MaxPeers: 1, NAT: nat.Any(), }, + NoUSB: true, }) } From 0f55f560b177b0fe1bc6df4d9f662c08291224f7 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Mon, 26 Nov 2018 16:28:10 +0100 Subject: [PATCH 19/61] set ANDROID_* envs --- _assets/ci/Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 676267206a5..ffb1da62e96 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -30,6 +30,10 @@ pipeline { STATUS_PATH = 'src/github.com/status-im/status-go' GOPATH = "${env.WORKSPACE}" PATH = "${env.PATH}:${env.GOPATH}/bin" + ANDROID_HOME = '/usr/lib/android-sdk' + ANDROID_SDK_ROOT = '/usr/lib/android-sdk' + ANDROID_NDK = '/usr/lib/android-ndk' + ANDROID_NDK_HOME = '/usr/lib/android-ndk' } stages { From 0b92aa6cfc65a93047144cf8e63c4bfa7061182d Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Wed, 28 Nov 2018 14:10:16 +0100 Subject: [PATCH 20/61] use ios/arm64 target for ios --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8b59dba6f31..af20c096b2f 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ statusgo-android: ##@cross-compile Build status-go for Android statusgo-ios: ##@cross-compile Build status-go for iOS @echo "Building status-go for iOS..." - @gomobile bind -target=ios -ldflags="-s -w" -o build/bin/Statusgo.framework github.com/status-im/status-go/mobile + @gomobile bind -target=ios/arm64 -ldflags="-s -w" -o build/bin/Statusgo.framework github.com/status-im/status-go/mobile @echo "iOS framework cross compilation done in build/bin/Statusgo.framework" statusgo-library: ##@cross-compile Build status-go as static library for current platform From 8177afadb1e6d2c2bdb90e0fb0956c7b3e2aca40 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Tue, 4 Dec 2018 13:02:27 +0100 Subject: [PATCH 21/61] rename package from `status` to `statusgo` to generate the iOS framework correctly. --- mobile/README.md | 8 ++++++-- mobile/response.go | 2 +- mobile/response_test.go | 2 +- mobile/status.go | 2 +- mobile/types.go | 2 +- mobile/utils.go | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mobile/README.md b/mobile/README.md index e60dee94287..87889844d70 100644 --- a/mobile/README.md +++ b/mobile/README.md @@ -2,6 +2,10 @@ Package mobile implements [gomobile](https://github.com/golang/mobile) bindings for status-go. Current implementation servers as a drop-in replacement for `lib` package. +The framework name is generated from the package name, hence these things are done intentionally: +(1) this package's name isn't equal to the directory name (`statusgo` vs `mobile` respectively); +(2) this package name is `statusgo` and not `status` which produces the right framework name. + # Usage For properly using this package, please refer to Makefile in the root of `status-go` directory. @@ -13,14 +17,14 @@ To manually build library, run following commands: ``` gomobile bind -v -target=ios -ldflags="-s -w" github.com/status-im/status-go/mobile ``` -This will produce `Status.framework` file in the current directory, which can be used in iOS project. +This will produce `Statusgo.framework` file in the current directory, which can be used in iOS project. ### Android ``` gomobile bind -v -target=android -ldflags="-s -w" github.com/status-im/status-go/mobile ``` -This will generate `Status.aar` file in the current dir. +This will generate `Statusgo.aar` file in the current dir. # Notes diff --git a/mobile/response.go b/mobile/response.go index 850124456cb..0614214c74e 100644 --- a/mobile/response.go +++ b/mobile/response.go @@ -1,4 +1,4 @@ -package status +package statusgo import ( "encoding/json" diff --git a/mobile/response_test.go b/mobile/response_test.go index c417d853726..a697cce0fa1 100644 --- a/mobile/response_test.go +++ b/mobile/response_test.go @@ -1,4 +1,4 @@ -package status +package statusgo import ( "errors" diff --git a/mobile/status.go b/mobile/status.go index 2b09d4aea68..fcc4f23fa7e 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -1,4 +1,4 @@ -package status +package statusgo import ( "encoding/json" diff --git a/mobile/types.go b/mobile/types.go index 9ba89d19ee7..4bc1768c3a5 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -1,4 +1,4 @@ -package status +package statusgo import ( "bytes" diff --git a/mobile/utils.go b/mobile/utils.go index ec56ef83dff..0837e2ad40e 100644 --- a/mobile/utils.go +++ b/mobile/utils.go @@ -1,4 +1,4 @@ -package status +package statusgo import ( "encoding/json" From 145192db4936b5a4adef3b404e9dbbbb3cb59b40 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Tue, 4 Dec 2018 16:31:16 +0100 Subject: [PATCH 22/61] fixup for iOS signals compiling --- signal/signals.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signal/signals.c b/signal/signals.c index a0c4bbd64bc..9237b76b571 100644 --- a/signal/signals.c +++ b/signal/signals.c @@ -1,4 +1,4 @@ -#if defined(IOS_DEPLOYMENT) +#if __has_include("objc/objc.h") // ====================================================================================== // iOS framework compilation using xgo // ====================================================================================== From 103eb853da65e293118f7a2f024de83e930b5dc2 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Tue, 4 Dec 2018 16:34:28 +0100 Subject: [PATCH 23/61] fix ios target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index af20c096b2f..8b59dba6f31 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ statusgo-android: ##@cross-compile Build status-go for Android statusgo-ios: ##@cross-compile Build status-go for iOS @echo "Building status-go for iOS..." - @gomobile bind -target=ios/arm64 -ldflags="-s -w" -o build/bin/Statusgo.framework github.com/status-im/status-go/mobile + @gomobile bind -target=ios -ldflags="-s -w" -o build/bin/Statusgo.framework github.com/status-im/status-go/mobile @echo "iOS framework cross compilation done in build/bin/Statusgo.framework" statusgo-library: ##@cross-compile Build status-go as static library for current platform From 648ba8428a85f1a8c14317c6b71f0b42a21440c2 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Tue, 4 Dec 2018 17:25:48 +0100 Subject: [PATCH 24/61] use Go code to propagate signals on mobile --- mobile/status.go | 7 +++++++ mobile/types.go | 4 ++++ signal/signals.go | 18 +++++++++++++++--- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/mobile/status.go b/mobile/status.go index fcc4f23fa7e..b888211dc87 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -449,6 +449,13 @@ func AppStateChange(state string) { statusBackend.AppStateChange(state) } +//SetMobileSignalHandler setup geth callback to notify about new signal +func SetMobileSignalHandler(handler SignalHandler) { + signal.SetMobileSignalHandler(func(data []byte) { + handler.HandleSignal(string(data)) + }) +} + // SetSignalEventCallback setup geth callback to notify about new signal func SetSignalEventCallback(cb unsafe.Pointer) { signal.SetSignalEventCallback(cb) diff --git a/mobile/types.go b/mobile/types.go index 4bc1768c3a5..863f5e64214 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -75,3 +75,7 @@ type NotifyResult struct { Status bool `json:"status"` Error string `json:"error,omitempty"` } + +type SignalHandler interface { + HandleSignal(string) +} diff --git a/signal/signals.go b/signal/signals.go index 14d14afd606..4c8f753b1eb 100644 --- a/signal/signals.go +++ b/signal/signals.go @@ -18,6 +18,14 @@ import ( "github.com/ethereum/go-ethereum/log" ) +type MobileSignalHandler func([]byte) + +var mobileSignalHandler MobileSignalHandler = nil + +func SetMobileSignalHandler(handler MobileSignalHandler) { + mobileSignalHandler = handler +} + // All general log messages in this package should be routed through this logger. var logger = log.New("package", "status-go/signal") @@ -44,9 +52,13 @@ func send(typ string, event interface{}) { return } - str := C.CString(string(data)) - C.StatusServiceSignalEvent(str) - C.free(unsafe.Pointer(str)) + if mobileSignalHandler != nil { + mobileSignalHandler(data) + } else { + str := C.CString(string(data)) + C.StatusServiceSignalEvent(str) + C.free(unsafe.Pointer(str)) + } } // NodeNotificationHandler defines a handler able to process incoming node events. From 2f6511059b6c4d013e5058469faf87a971f88acf Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Tue, 4 Dec 2018 17:49:20 +0100 Subject: [PATCH 25/61] clean up most of the C magic of signals package (only leftovers for Desktop) --- mobile/status.go | 7 +- signal/signals.c | 207 ---------------------------------------------- signal/signals.go | 15 +++- 3 files changed, 16 insertions(+), 213 deletions(-) diff --git a/mobile/status.go b/mobile/status.go index b888211dc87..dd05deeed7d 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -449,10 +449,13 @@ func AppStateChange(state string) { statusBackend.AppStateChange(state) } -//SetMobileSignalHandler setup geth callback to notify about new signal +// SetMobileSignalHandler setup geth callback to notify about new signal +// used for gomobile builds func SetMobileSignalHandler(handler SignalHandler) { signal.SetMobileSignalHandler(func(data []byte) { - handler.HandleSignal(string(data)) + if data != nil && len(data) > 0 { + handler.HandleSignal(string(data)) + } }) } diff --git a/signal/signals.c b/signal/signals.c index 9237b76b571..8274bea39f1 100644 --- a/signal/signals.c +++ b/signal/signals.c @@ -1,208 +1,3 @@ -#if __has_include("objc/objc.h") -// ====================================================================================== -// iOS framework compilation using xgo -// ====================================================================================== - -#include -#include -#include -#include -#include -#include - -static id statusServiceClassRef = nil; -static SEL statusServiceSelector = nil; - -static bool initLibrary() { - if (statusServiceClassRef == nil) { - statusServiceClassRef = objc_getClass("Status"); - if (statusServiceClassRef == nil) return false; - } - - if (statusServiceSelector == nil) { - statusServiceSelector = sel_getUid("signalEvent:"); - if (statusServiceSelector == nil) return false; - } - - return true; -} - - -/*! - * @brief Calls static method signalEvent of class GethService. - * - * @param jsonEvent - UTF8 string - * - * @note Definition of signalEvent method. - * + (void)signalEvent:(const char *)json - */ -bool StatusServiceSignalEvent(const char *jsonEvent) { - if (!initLibrary()) return false; - - void (*action)(id, SEL, const char *) = (void (*)(id, SEL, const char *)) objc_msgSend; - action(statusServiceClassRef, statusServiceSelector, jsonEvent); - - return true; -} - -void SetEventCallback(void *cb) { -} - -#elif defined(ANDROID_DEPLOYMENT) -// ====================================================================================== -// Android archive compilation using xgo -// ====================================================================================== - -#include -#include -#include - -bool StatusServiceSignalEvent(const char *jsonEvent); - -static JavaVM *gJavaVM = NULL; -static jclass JavaClassPtr_StatusService = NULL; -static jmethodID JavaMethodPtr_signalEvent = NULL; - -static bool JniLibraryInit(JNIEnv *env); - -/*! - * @brief Get interface to JNI. - * - * @return true if thread should be detached from JNI. - */ -static bool JniAttach(JNIEnv **env) { - jint status; - - if (gJavaVM == NULL) { - env = NULL; - } - - status = (*gJavaVM)->GetEnv(gJavaVM, (void **)env, JNI_VERSION_1_6); - if (status == JNI_EDETACHED) { - // attach thread to JNI - //(*gJavaVM)->AttachCurrentThread( gJavaVM, (void **)env, NULL ); // Oracle JNI API - (*gJavaVM)->AttachCurrentThread(gJavaVM, env, NULL); // Android JNI API - return true; - } else if (status != JNI_OK) { - return false; - } - - return false; -} - -/*! - * @brief The VM calls JNI_OnLoad when the native library is loaded. - */ -JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved) { - bool detach; - JNIEnv *env; - int result = JNI_VERSION_1_6; - - gJavaVM = vm; - - // attach thread to JNI - detach = JniAttach(&env); - if (env == NULL) { - // failed - gJavaVM = NULL; - return 0; - } - - if (!JniLibraryInit(env)) { - // fail loading of JNI library - result = 0; - } - - if (detach) { - // detach thread from JNI - (*gJavaVM)->DetachCurrentThread(gJavaVM); - } - - if (result != JNI_VERSION_1_6) { - gJavaVM = NULL; - } - - return result; -} - -/*! - * @brief Initialize library. - */ -bool JniLibraryInit(JNIEnv *env) { - int i; - - JavaClassPtr_StatusService = (*env)->FindClass(env, "im/status/ethereum/module/StatusService"); - if (JavaClassPtr_StatusService == NULL) return false; - - JavaClassPtr_StatusService = (jclass)(*env)->NewGlobalRef(env, JavaClassPtr_StatusService); - if (JavaClassPtr_StatusService == NULL) return false; - - struct { - bool bStatic; - jclass classPtr; - jmethodID *methodPtr; - const char *methodId; - const char *params; - } javaMethodDescriptors[] = { - { - true, - JavaClassPtr_StatusService, - &JavaMethodPtr_signalEvent, // &JavaMethodPtr_someNonStaticMethod - "signalEvent", // someNonStaticMethod - "(Ljava/lang/String;)V" - }, - }; - - for (i = 0; i < sizeof(javaMethodDescriptors) / sizeof(javaMethodDescriptors[0]); i++) { - if (javaMethodDescriptors[i].bStatic) { - *(javaMethodDescriptors[i].methodPtr) = (*env)->GetStaticMethodID( - env, javaMethodDescriptors[i].classPtr, javaMethodDescriptors[i].methodId, javaMethodDescriptors[i].params); - } else { - *(javaMethodDescriptors[i].methodPtr) = (*env)->GetMethodID( - env, javaMethodDescriptors[i].classPtr, javaMethodDescriptors[i].methodId, javaMethodDescriptors[i].params); - } - - if (*(javaMethodDescriptors[i].methodPtr) == NULL) return false; - } - - return true; -} - -/*! - * @brief Calls static method signalEvent of class im.status.ethereum.module.StatusService. - * - * @param jsonEvent - UTF8 string - */ -bool StatusServiceSignalEvent(const char *jsonEvent) { - bool detach; - JNIEnv *env; - - // attach thread to JNI - detach = JniAttach( &env ); - if (env == NULL) { // failed - return false; - } - - jstring javaJsonEvent = NULL; - if (jsonEvent != NULL) { - javaJsonEvent = (*env)->NewStringUTF(env, jsonEvent); - } - - (*env)->CallStaticVoidMethod(env, JavaClassPtr_StatusService, JavaMethodPtr_signalEvent, javaJsonEvent); - - if (javaJsonEvent != NULL) (*env)->DeleteLocalRef(env, javaJsonEvent); - - if (detach) { // detach thread from JNI - (*gJavaVM)->DetachCurrentThread(gJavaVM); - } - - return true; -} - -void SetEventCallback(void *cb) { -} - -#else // ====================================================================================== // cgo compilation (for desktop platforms and local tests) // ====================================================================================== @@ -228,5 +23,3 @@ bool StatusServiceSignalEvent(const char *jsonEvent) { void SetEventCallback(void *cb) { gCallback = (callback)cb; } - -#endif diff --git a/signal/signals.go b/signal/signals.go index 4c8f753b1eb..87e486ab3b5 100644 --- a/signal/signals.go +++ b/signal/signals.go @@ -18,14 +18,12 @@ import ( "github.com/ethereum/go-ethereum/log" ) +// MobileSignalHandler is a simple callback function that gets called when any signal is received type MobileSignalHandler func([]byte) +// storing the current signal handler here var mobileSignalHandler MobileSignalHandler = nil -func SetMobileSignalHandler(handler MobileSignalHandler) { - mobileSignalHandler = handler -} - // All general log messages in this package should be routed through this logger. var logger = log.New("package", "status-go/signal") @@ -52,8 +50,10 @@ func send(typ string, event interface{}) { return } + // if a go implementation of signal handler is set, we use it if mobileSignalHandler != nil { mobileSignalHandler(data) + // ... and fallback to C implementation otherwise } else { str := C.CString(string(data)) C.StatusServiceSignalEvent(str) @@ -105,7 +105,14 @@ func TriggerTestSignal() { C.free(unsafe.Pointer(str)) } +// SetMobileSignalHandler sets new handler for geth events +// this function uses pure go implementation +func SetMobileSignalHandler(handler MobileSignalHandler) { + mobileSignalHandler = handler +} + // SetSignalEventCallback set callback +// this function uses C implementation (see `signals.c` file) func SetSignalEventCallback(cb unsafe.Pointer) { C.SetEventCallback(cb) } From 5866167428bb0a66dd0d97fb7f788730063ac58b Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Wed, 5 Dec 2018 07:24:29 +0100 Subject: [PATCH 26/61] fix target for android --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8b59dba6f31..bc47ae65eda 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ statusgo-cross: statusgo-android statusgo-ios statusgo-android: ##@cross-compile Build status-go for Android @echo "Building status-go for Android..." - @gomobile bind -target=android/arm -ldflags="-s -w" -o build/bin/statusgo.aar github.com/status-im/status-go/mobile + @gomobile bind -target=android -ldflags="-s -w" -o build/bin/statusgo.aar github.com/status-im/status-go/mobile @echo "Android cross compilation done in build/bin/statusgo.aar" statusgo-ios: ##@cross-compile Build status-go for iOS @@ -137,7 +137,7 @@ docker-image: ##@docker Build docker image (use DOCKER_IMAGE_NAME to set the ima -t $(DOCKER_IMAGE_NAME):latest bootnode-image: - @echo "Building docker image for bootnode..."` + @echo "Building docker image for bootnode..." docker build --file _assets/build/Dockerfile-bootnode . \ --build-arg "build_tags=$(BUILD_TAGS)" \ --build-arg "build_flags=$(BUILD_FLAGS)" \ From ff429343ad104099967dff03c8ad62057acff954 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Wed, 5 Dec 2018 07:30:49 +0100 Subject: [PATCH 27/61] move comment --- signal/signals.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signal/signals.go b/signal/signals.go index 87e486ab3b5..2804df79a2b 100644 --- a/signal/signals.go +++ b/signal/signals.go @@ -50,11 +50,11 @@ func send(typ string, event interface{}) { return } - // if a go implementation of signal handler is set, we use it + // If a Go implementation of signal handler is set, let's use it. if mobileSignalHandler != nil { mobileSignalHandler(data) - // ... and fallback to C implementation otherwise } else { + // ...and fallback to C implementation otherwise. str := C.CString(string(data)) C.StatusServiceSignalEvent(str) C.free(unsafe.Pointer(str)) From 3bccb96e468e3462cf327373900a75427a92f743 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Wed, 5 Dec 2018 12:49:12 +0100 Subject: [PATCH 28/61] make NofifyUsers work again (after rebase) --- mobile/status.go | 47 +++++++++++++++-------------------------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/mobile/status.go b/mobile/status.go index dd05deeed7d..5da19114f49 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -6,7 +6,6 @@ import ( "os" "unsafe" - "github.com/NaySoftware/go-fcm" "github.com/ethereum/go-ethereum/log" "github.com/status-im/status-go/api" "github.com/status-im/status-go/logutils" @@ -387,49 +386,33 @@ func makeJSONResponse(err error) string { } // NotifyUsers sends push notifications by given tokens. -func NotifyUsers(message, payloadJSON, tokensArray string) (outCBytes string) { - var ( - err error - outBytes []byte - ) - errString := "" +// TODO: remove unusedField +func NotifyUsers(unusedField, payloadJSON, tokensArray string) string { + makeResponse := func(err error) string { + result := NotifyResult{} - defer func() { - out := NotifyResult{ - Status: err == nil, - Error: errString, + result.Status = err == nil + if err != nil { + result.Error = err.Error() } - outBytes, err = json.Marshal(out) + resultJSON, err := json.Marshal(result) + if err != nil { logger.Error("failed to marshal Notify output", "error", err) - outCBytes = makeJSONResponse(err) - return + return makeJSONResponse(err) } - outCBytes = string(outBytes) - }() - - tokens, err := ParseJSONArray(tokensArray) - if err != nil { - errString = err.Error() - return + return string(resultJSON) } - var payload fcm.NotificationPayload - err = json.Unmarshal([]byte(payloadJSON), &payload) - if err != nil { - errString = err.Error() - return - } - - err = statusBackend.NotifyUsers(message, payload, tokens...) + tokens, err := ParseJSONArray(tokensArray) if err != nil { - errString = err.Error() - return + return makeResponse(err) } - return + err = statusBackend.NotifyUsers(payloadJSON, tokens...) + return makeResponse(err) } // AddPeer adds an enode as a peer. From 441bd5a55b12ff9d8810210fa400b71468058baa Mon Sep 17 00:00:00 2001 From: Andrea Maria Piana Date: Wed, 5 Dec 2018 09:22:49 +0100 Subject: [PATCH 29/61] Notify user when the device is missing (#1298) * Notify user when the device is missing * Update services/shhext/chat/encryption.go Co-Authored-By: cammellos --- VERSION | 2 +- _assets/ci/Jenkinsfile | 4 +-- services/shhext/api.go | 15 +++-------- services/shhext/chat/encryption.go | 7 +++--- services/shhext/chat/encryption_test.go | 33 +++++++++++++++++++++++++ services/shhext/chat/protocol.go | 32 ++++++++++-------------- services/shhext/chat/protocol_test.go | 9 ++++--- services/shhext/service.go | 10 +++++++- 8 files changed, 70 insertions(+), 42 deletions(-) diff --git a/VERSION b/VERSION index e1edef6db5a..539c147a422 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.18.0-beta +0.18.1-beta diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index f5efd317f80..8c0afb75359 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -83,7 +83,7 @@ pipeline { usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN' ]]) { - sh "yes | make release release_branch=${gitBranch}" + sh "yes | make release RELEASE_BRANCH=${lib.gitBranch()}" } sh 'make clean-release' } @@ -151,7 +151,7 @@ pipeline { usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN' ]]) { - sh "yes | make release release_branch=${gitBranch}" + sh "yes | make release RELEASE_BRANCH=${lib.gitBranch()}" } sh 'make clean-release' } diff --git a/services/shhext/api.go b/services/shhext/api.go index 9d31a575ae9..a6bd10ecb31 100644 --- a/services/shhext/api.go +++ b/services/shhext/api.go @@ -416,11 +416,9 @@ func (api *PublicAPI) processPFSMessage(msg *whisper.Message) error { response, err := api.service.protocol.HandleMessage(privateKey, publicKey, msg.Payload) - handler := EnvelopeSignalHandler{} - // Notify that someone tried to contact us using an invalid bundle - if err == chat.ErrSessionNotFound { - api.log.Warn("Session not found, sending signal", "err", err) + if err == chat.ErrDeviceNotFound && privateKey.PublicKey != *publicKey { + api.log.Warn("Device not found, sending signal", "err", err) keyString := fmt.Sprintf("0x%x", crypto.FromECDSAPub(publicKey)) handler := EnvelopeSignalHandler{} handler.DecryptMessageFailed(keyString) @@ -432,14 +430,7 @@ func (api *PublicAPI) processPFSMessage(msg *whisper.Message) error { } // Add unencrypted payload - msg.Payload = response.Message - - // Notify of added bundles - if response.AddedBundles != nil { - for _, bundle := range response.AddedBundles { - handler.BundleAdded(bundle[0], bundle[1]) - } - } + msg.Payload = response return nil } diff --git a/services/shhext/chat/encryption.go b/services/shhext/chat/encryption.go index ab20abf615c..2a0933f8a4f 100644 --- a/services/shhext/chat/encryption.go +++ b/services/shhext/chat/encryption.go @@ -18,6 +18,7 @@ import ( ) var ErrSessionNotFound = errors.New("session not found") +var ErrDeviceNotFound = errors.New("device not found") // If we have no bundles, we use a constant so that the message can reach any device. const noInstallationID = "none" @@ -53,7 +54,7 @@ func DefaultEncryptionServiceConfig(installationID string) EncryptionServiceConf MaxSkip: 1000, MaxKeep: 3000, MaxMessageKeysPerSession: 2000, - BundleRefreshInterval: 14 * 24 * 60 * 60 * 1000, + BundleRefreshInterval: 6 * 60 * 60 * 1000, InstallationID: installationID, } } @@ -238,8 +239,8 @@ func (s *EncryptionService) DecryptPayload(myIdentityKey *ecdsa.PrivateKey, thei } // We should not be sending a signal if it's coming from us, as we receive our own messages - if msg == nil { - return nil, ErrSessionNotFound + if msg == nil && *theirIdentityKey != myIdentityKey.PublicKey { + return nil, ErrDeviceNotFound } payload := msg.GetPayload() diff --git a/services/shhext/chat/encryption_test.go b/services/shhext/chat/encryption_test.go index 7c5ee825a50..ee309cedaa4 100644 --- a/services/shhext/chat/encryption_test.go +++ b/services/shhext/chat/encryption_test.go @@ -770,6 +770,39 @@ func (s *EncryptionServiceTestSuite) TestBundleNotExisting() { s.Equal(ErrSessionNotFound, err) } +// Device is not included in the bundle +func (s *EncryptionServiceTestSuite) TestDeviceNotIncluded() { + bobDevice2InstallationID := "bob2" + + bobKey, err := crypto.GenerateKey() + s.Require().NoError(err) + + aliceKey, err := crypto.GenerateKey() + s.Require().NoError(err) + + // Create a bundle without saving it + bobBundleContainer, err := NewBundleContainer(bobKey, bobDevice2InstallationID) + s.Require().NoError(err) + + err = SignBundle(bobKey, bobBundleContainer) + s.Require().NoError(err) + + bobBundle := bobBundleContainer.GetBundle() + + // We add bob bundle + _, err = s.alice.ProcessPublicBundle(aliceKey, bobBundle) + s.Require().NoError(err) + + // Alice sends a message + aliceMessage, err := s.alice.EncryptPayload(&bobKey.PublicKey, aliceKey, []byte("does not matter")) + s.Require().NoError(err) + + // Bob receives the message, and returns a bundlenotfound error + _, err = s.bob.DecryptPayload(bobKey, &aliceKey.PublicKey, aliceInstallationID, aliceMessage) + s.Require().Error(err) + s.Equal(ErrDeviceNotFound, err) +} + // A new bundle has been received func (s *EncryptionServiceTestSuite) TestRefreshedBundle() { diff --git a/services/shhext/chat/protocol.go b/services/shhext/chat/protocol.go index 9237b1b2280..2e990699107 100644 --- a/services/shhext/chat/protocol.go +++ b/services/shhext/chat/protocol.go @@ -9,21 +9,18 @@ import ( ) type ProtocolService struct { - log log.Logger - encryption *EncryptionService - Enabled bool -} - -type HandleMessageResponse struct { - AddedBundles []IdentityAndIDPair - Message []byte + log log.Logger + encryption *EncryptionService + addedBundlesHandler func([]IdentityAndIDPair) + Enabled bool } // NewProtocolService creates a new ProtocolService instance -func NewProtocolService(encryption *EncryptionService) *ProtocolService { +func NewProtocolService(encryption *EncryptionService, addedBundlesHandler func([]IdentityAndIDPair)) *ProtocolService { return &ProtocolService{ - log: log.New("package", "status-go/services/sshext.chat"), - encryption: encryption, + log: log.New("package", "status-go/services/sshext.chat"), + encryption: encryption, + addedBundlesHandler: addedBundlesHandler, } } @@ -126,13 +123,11 @@ func (p *ProtocolService) DisableInstallation(myIdentityKey *ecdsa.PublicKey, in } // HandleMessage unmarshals a message and processes it, decrypting it if it is a 1:1 message. -func (p *ProtocolService) HandleMessage(myIdentityKey *ecdsa.PrivateKey, theirPublicKey *ecdsa.PublicKey, payload []byte) (*HandleMessageResponse, error) { +func (p *ProtocolService) HandleMessage(myIdentityKey *ecdsa.PrivateKey, theirPublicKey *ecdsa.PublicKey, payload []byte) ([]byte, error) { if p.encryption == nil { return nil, errors.New("encryption service not initialized") } - response := &HandleMessageResponse{} - // Unmarshal message protocolMessage := &ProtocolMessage{} @@ -147,14 +142,14 @@ func (p *ProtocolService) HandleMessage(myIdentityKey *ecdsa.PrivateKey, theirPu if err != nil { return nil, err } - response.AddedBundles = addedBundles + + p.addedBundlesHandler(addedBundles) } // Check if it's a public message if publicMessage := protocolMessage.GetPublicMessage(); publicMessage != nil { - response.Message = publicMessage // Nothing to do, as already in cleartext - return response, nil + return publicMessage, nil } // Decrypt message @@ -163,9 +158,8 @@ func (p *ProtocolService) HandleMessage(myIdentityKey *ecdsa.PrivateKey, theirPu if err != nil { return nil, err } - response.Message = message - return response, nil + return message, nil } // Return error diff --git a/services/shhext/chat/protocol_test.go b/services/shhext/chat/protocol_test.go index 1eee68db26a..c1f9c04f643 100644 --- a/services/shhext/chat/protocol_test.go +++ b/services/shhext/chat/protocol_test.go @@ -38,8 +38,10 @@ func (s *ProtocolServiceTestSuite) SetupTest() { panic(err) } - s.alice = NewProtocolService(NewEncryptionService(alicePersistence, DefaultEncryptionServiceConfig("1"))) - s.bob = NewProtocolService(NewEncryptionService(bobPersistence, DefaultEncryptionServiceConfig("2"))) + addedBundlesHandler := func(addedBundles []IdentityAndIDPair) {} + + s.alice = NewProtocolService(NewEncryptionService(alicePersistence, DefaultEncryptionServiceConfig("1")), addedBundlesHandler) + s.bob = NewProtocolService(NewEncryptionService(bobPersistence, DefaultEncryptionServiceConfig("2")), addedBundlesHandler) } func (s *ProtocolServiceTestSuite) TestBuildDirectMessage() { @@ -102,9 +104,8 @@ func (s *ProtocolServiceTestSuite) TestBuildAndReadDirectMessage() { s.NoError(err) // Bob is able to decrypt the message - response, err := s.bob.HandleMessage(bobKey, &aliceKey.PublicKey, marshaledMsg[&bobKey.PublicKey]) + unmarshaledMsg, err := s.bob.HandleMessage(bobKey, &aliceKey.PublicKey, marshaledMsg[&bobKey.PublicKey]) s.NoError(err) - unmarshaledMsg := response.Message s.NotNil(unmarshaledMsg) diff --git a/services/shhext/service.go b/services/shhext/service.go index 00d51a0efa5..59505328427 100644 --- a/services/shhext/service.go +++ b/services/shhext/service.go @@ -118,7 +118,15 @@ func (s *Service) InitProtocol(address string, password string) error { if err != nil { return err } - s.protocol = chat.NewProtocolService(chat.NewEncryptionService(persistence, chat.DefaultEncryptionServiceConfig(s.installationID))) + + addedBundlesHandler := func(addedBundles []chat.IdentityAndIDPair) { + handler := EnvelopeSignalHandler{} + for _, bundle := range addedBundles { + handler.BundleAdded(bundle[0], bundle[1]) + } + } + + s.protocol = chat.NewProtocolService(chat.NewEncryptionService(persistence, chat.DefaultEncryptionServiceConfig(s.installationID)), addedBundlesHandler) return nil } From bf2c932d671548cb052203c840f28addc98b9900 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 5 Dec 2018 15:57:05 +0200 Subject: [PATCH 30/61] Mail peer store and connection manager (#1295) This change implements connection manager that monitors 3 types of events: 1. update of the selected mail servers 2. disconnect from a mail server 3. errors for requesting mail history When selected mail servers provided we will try to connect with as many as possible, and later disconnect the surplus. For example if we want to connect with one mail server and 3 were selected, we try to connect with all (3), and later disconnect with 2. It will to establish connection with live mail server faster. If mail server disconnects we will choose any other mail server from the list of selected. Unless we have only one mail server. In such case we don't have any other choice and we will leave things as is. If request for history was expired we will disconnect such peer and try to find another one. We will follow same rules as described above. We will have two components that will rely on this logic: 1. requesting history If target peer is provided we will use that peer, otherwise we will request history from any selected mail server that is connected at the time of request. 2. confirmation from selected mail server Confirmation from any selected mail server will bee used to send a feedback that envelope was sent. I will add several extensions, but probably in separate PRs: 1. prioritize connection with mail server that was used before reboot 2. disconnect from mail servers if history request wasn't expired but failed. 3. wait some time in RequestsMessage RPC to establish connection with any mail server Currently this feature is hidden, as certain changes will be necessary in status-react. partially implements: https://github.com/status-im/status-go/issues/1285 --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- cmd/node-canary/main.go | 4 +- services/shhext/api.go | 19 +- services/shhext/mailservers/connmanager.go | 202 +++++++++++ .../shhext/mailservers/connmanager_test.go | 313 ++++++++++++++++++ services/shhext/mailservers/peerstore.go | 67 ++++ services/shhext/mailservers/peerstore_test.go | 77 +++++ services/shhext/service.go | 271 ++------------- services/shhext/service_test.go | 177 ++-------- services/shhext/tracker.go | 218 ++++++++++++ services/shhext/tracker_test.go | 144 ++++++++ t/e2e/whisper/mailservice_test.go | 4 +- .../status-im/whisper/whisperv6/events.go | 6 +- .../status-im/whisper/whisperv6/whisper.go | 30 +- 15 files changed, 1135 insertions(+), 405 deletions(-) create mode 100644 services/shhext/mailservers/connmanager.go create mode 100644 services/shhext/mailservers/connmanager_test.go create mode 100644 services/shhext/mailservers/peerstore.go create mode 100644 services/shhext/mailservers/peerstore_test.go create mode 100644 services/shhext/tracker.go create mode 100644 services/shhext/tracker_test.go diff --git a/Gopkg.lock b/Gopkg.lock index fe6f6d2f693..c6ec200fb12 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -822,12 +822,12 @@ revision = "fbcc46a78cd43fef95a110df664aab513116a850" [[projects]] - digest = "1:6cb252f27feb57ef0e8406556c259d903c0ecff2ab0d2200ca85773b3561777d" + digest = "1:5c62af344925b846377386dec72e06eb3e1e15222542b3d22fe0f0da75c7f090" name = "github.com/status-im/whisper" packages = ["whisperv6"] pruneopts = "NUT" - revision = "76c24476436f0cf832021be98316a4ee62cc83cc" - version = "v1.4.0" + revision = "96d2199ed511430c642d877afe7bacaac5f37426" + version = "v1.4.1" [[projects]] digest = "1:572c783a763db6383aca3179976eb80e4c900f52eba56cba8bb2e3cea7ce720e" diff --git a/Gopkg.toml b/Gopkg.toml index e37bc60567d..254703bb294 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -29,7 +29,7 @@ [[constraint]] name = "github.com/status-im/whisper" - version = "=v1.4.0" + version = "=v1.4.1" [[override]] name = "github.com/golang/protobuf" diff --git a/cmd/node-canary/main.go b/cmd/node-canary/main.go index 9bc5b04c4bb..9df9fe6fa09 100644 --- a/cmd/node-canary/main.go +++ b/cmd/node-canary/main.go @@ -173,7 +173,7 @@ func verifyMailserverBehavior(mailserverNode *enode.Node) { Limit: 1, Topic: topic, SymKeyID: mailServerKeyID, - Timeout: time.Duration(*timeout) * time.Second, + Timeout: time.Duration(*timeout), }) if err != nil { logger.Error("Error requesting historic messages from mailserver", "error", err) @@ -333,6 +333,8 @@ func waitForMailServerResponse(events chan whisper.EnvelopeEvent, requestID comm func decodeMailServerResponse(event whisper.EnvelopeEvent) (*whisper.MailServerResponse, error) { switch event.Event { + case whisper.EventMailServerRequestSent: + return nil, nil case whisper.EventMailServerRequestCompleted: resp, ok := event.Data.(*whisper.MailServerResponse) if !ok { diff --git a/services/shhext/api.go b/services/shhext/api.go index a6bd10ecb31..b3a69e029a0 100644 --- a/services/shhext/api.go +++ b/services/shhext/api.go @@ -17,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/status-im/status-go/services/shhext/chat" + "github.com/status-im/status-go/services/shhext/mailservers" whisper "github.com/status-im/whisper/whisperv6" ) @@ -150,6 +151,13 @@ func (api *PublicAPI) Post(ctx context.Context, req whisper.NewMessage) (hash he return hash, err } +func (api *PublicAPI) getPeer(rawurl string) (*enode.Node, error) { + if len(rawurl) == 0 { + return mailservers.GetFirstConnected(api.service.server, api.service.peerStore) + } + return enode.ParseV4(rawurl) +} + // RequestMessages sends a request for historic messages to a MailServer. func (api *PublicAPI) RequestMessages(_ context.Context, r MessagesRequest) (hexutil.Bytes, error) { api.log.Info("RequestMessages", "request", r) @@ -161,9 +169,7 @@ func (api *PublicAPI) RequestMessages(_ context.Context, r MessagesRequest) (hex return nil, fmt.Errorf("Query range is invalid: from > to (%d > %d)", r.From, r.To) } - var err error - - mailServerNode, err := enode.ParseV4(r.MailServerPeer) + mailServerNode, err := api.getPeer(r.MailServerPeer) if err != nil { return nil, fmt.Errorf("%v: %v", ErrInvalidMailServerPeer, err) } @@ -199,13 +205,10 @@ func (api *PublicAPI) RequestMessages(_ context.Context, r MessagesRequest) (hex return nil, err } - hash := envelope.Hash() - if err := shh.RequestHistoricMessages(mailServerNode.ID().Bytes(), envelope); err != nil { + if err := shh.RequestHistoricMessagesWithTimeout(mailServerNode.ID().Bytes(), envelope, r.Timeout*time.Second); err != nil { return nil, err } - - api.service.tracker.AddRequest(hash, time.After(r.Timeout*time.Second)) - + hash := envelope.Hash() return hash[:], nil } diff --git a/services/shhext/mailservers/connmanager.go b/services/shhext/mailservers/connmanager.go new file mode 100644 index 00000000000..905d0c01fea --- /dev/null +++ b/services/shhext/mailservers/connmanager.go @@ -0,0 +1,202 @@ +package mailservers + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/status-im/whisper/whisperv6" +) + +const ( + peerEventsBuffer = 10 // sufficient buffer to avoid blocking a p2p feed. + whisperEventsBuffer = 20 // sufficient buffer to avod blocking a whisper envelopes feed. +) + +// PeerAdderRemover is an interface for adding or removing peers. +type PeerAdderRemover interface { + AddPeer(node *enode.Node) + RemovePeer(node *enode.Node) +} + +// PeerEventsSubscriber interface to subscribe for p2p.PeerEvent's. +type PeerEventsSubscriber interface { + SubscribeEvents(chan *p2p.PeerEvent) event.Subscription +} + +// EnvelopeEventSubscbriber interface to subscribe for whisper.EnvelopeEvent's. +type EnvelopeEventSubscbriber interface { + SubscribeEnvelopeEvents(chan<- whisperv6.EnvelopeEvent) event.Subscription +} + +type p2pServer interface { + PeerAdderRemover + PeerEventsSubscriber +} + +// NewConnectionManager creates an instance of ConnectionManager. +func NewConnectionManager(server p2pServer, whisper EnvelopeEventSubscbriber, target int) *ConnectionManager { + return &ConnectionManager{ + server: server, + whisper: whisper, + connectedTarget: target, + notifications: make(chan []*enode.Node), + } +} + +// ConnectionManager manages keeps target of peers connected. +type ConnectionManager struct { + wg sync.WaitGroup + quit chan struct{} + + server p2pServer + whisper EnvelopeEventSubscbriber + + notifications chan []*enode.Node + connectedTarget int +} + +// Notify sends a non-blocking notification about new nodes. +func (ps *ConnectionManager) Notify(nodes []*enode.Node) { + ps.wg.Add(1) + go func() { + select { + case ps.notifications <- nodes: + case <-ps.quit: + } + ps.wg.Done() + }() + +} + +// Start subscribes to a p2p server and handles new peers and state updates for those peers. +func (ps *ConnectionManager) Start() { + ps.quit = make(chan struct{}) + ps.wg.Add(1) + go func() { + current := map[enode.ID]*enode.Node{} + connected := map[enode.ID]struct{}{} + events := make(chan *p2p.PeerEvent, peerEventsBuffer) + sub := ps.server.SubscribeEvents(events) + whisperEvents := make(chan whisperv6.EnvelopeEvent, whisperEventsBuffer) + whisperSub := ps.whisper.SubscribeEnvelopeEvents(whisperEvents) + requests := map[common.Hash]struct{}{} + for { + select { + case <-ps.quit: + sub.Unsubscribe() + whisperSub.Unsubscribe() + ps.wg.Done() + return + case err := <-sub.Err(): + log.Error("retry after error subscribing to p2p events", "error", err) + sub = ps.server.SubscribeEvents(events) + case err := <-whisperSub.Err(): + log.Error("retry after error suscribing to whisper events", "error", err) + whisperSub = ps.whisper.SubscribeEnvelopeEvents(whisperEvents) + case newNodes := <-ps.notifications: + replacement := map[enode.ID]*enode.Node{} + for _, n := range newNodes { + replacement[n.ID()] = n + } + replaceNodes(ps.server, ps.connectedTarget, connected, current, replacement) + current = replacement + case ev := <-events: + switch ev.Type { + case p2p.PeerEventTypeAdd: + log.Debug("connected to a mailserver", "address", ev.Peer) + nodeAdded(ps.server, ev.Peer, ps.connectedTarget, connected, current) + case p2p.PeerEventTypeDrop: + log.Debug("mailserver disconnected", "address", ev.Peer) + nodeDisconnected(ps.server, ev.Peer, ps.connectedTarget, connected, current) + } + case ev := <-whisperEvents: + // TODO what about completed but with error? what about expired envelopes? + switch ev.Event { + case whisperv6.EventMailServerRequestSent: + requests[ev.Hash] = struct{}{} + case whisperv6.EventMailServerRequestCompleted: + delete(requests, ev.Hash) + case whisperv6.EventMailServerRequestExpired: + _, exist := requests[ev.Hash] + if !exist { + continue + } + log.Debug("request to a mail server expired, disconncet a peer", "address", ev.Peer) + nodeDisconnected(ps.server, ev.Peer, ps.connectedTarget, connected, current) + } + } + } + }() +} + +// Stop gracefully closes all background goroutines and waits until they finish. +func (ps *ConnectionManager) Stop() { + if ps.quit == nil { + return + } + select { + case <-ps.quit: + return + default: + } + close(ps.quit) + ps.wg.Wait() + ps.quit = nil +} + +func replaceNodes(srv PeerAdderRemover, target int, connected map[enode.ID]struct{}, old, new map[enode.ID]*enode.Node) { + for nid, n := range old { + if _, exist := new[nid]; !exist { + if _, exist := connected[nid]; exist { + delete(connected, nid) + } + srv.RemovePeer(n) + } + } + if len(connected) < target { + for _, n := range new { + srv.AddPeer(n) + } + } +} + +func nodeAdded(srv PeerAdderRemover, peer enode.ID, target int, connected map[enode.ID]struct{}, nodes map[enode.ID]*enode.Node) { + n, exist := nodes[peer] + if !exist { + return + } + if len(connected) == target { + srv.RemovePeer(n) + } else { + connected[n.ID()] = struct{}{} + } +} + +func nodeDisconnected(srv PeerAdderRemover, peer enode.ID, target int, connected map[enode.ID]struct{}, nodes map[enode.ID]*enode.Node) { + n, exist := nodes[peer] // unrelated event + if !exist { + return + } + _, exist = connected[peer] // check if already disconnected + if !exist { + return + } + if len(nodes) == 1 { // keep node connected if we don't have another choice + return + } + srv.RemovePeer(n) // remove peer permanently, otherwise p2p.Server will try to reconnect + delete(connected, peer) + if len(connected) < target { // try to connect with any other selected (but not connected) node + for nid, n := range nodes { + _, exist := connected[nid] + if exist || peer == nid { + continue + } + srv.AddPeer(n) + } + } +} diff --git a/services/shhext/mailservers/connmanager_test.go b/services/shhext/mailservers/connmanager_test.go new file mode 100644 index 00000000000..0e8f45144f2 --- /dev/null +++ b/services/shhext/mailservers/connmanager_test.go @@ -0,0 +1,313 @@ +package mailservers + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/status-im/status-go/t/utils" + "github.com/status-im/whisper/whisperv6" + "github.com/stretchr/testify/require" +) + +type fakePeerEvents struct { + mu sync.Mutex + nodes map[enode.ID]struct{} + input chan *p2p.PeerEvent +} + +func (f *fakePeerEvents) Nodes() []enode.ID { + f.mu.Lock() + rst := make([]enode.ID, 0, len(f.nodes)) + for n := range f.nodes { + rst = append(rst, n) + } + f.mu.Unlock() + return rst +} + +func (f *fakePeerEvents) AddPeer(node *enode.Node) { + f.mu.Lock() + f.nodes[node.ID()] = struct{}{} + f.mu.Unlock() + if f.input == nil { + return + } + f.input <- &p2p.PeerEvent{ + Peer: node.ID(), + Type: p2p.PeerEventTypeAdd, + } +} + +func (f *fakePeerEvents) RemovePeer(node *enode.Node) { + f.mu.Lock() + delete(f.nodes, node.ID()) + f.mu.Unlock() + if f.input == nil { + return + } + f.input <- &p2p.PeerEvent{ + Peer: node.ID(), + Type: p2p.PeerEventTypeDrop, + } +} + +func newFakePeerAdderRemover() *fakePeerEvents { + return &fakePeerEvents{nodes: map[enode.ID]struct{}{}} +} + +func (f *fakePeerEvents) SubscribeEvents(output chan *p2p.PeerEvent) event.Subscription { + return event.NewSubscription(func(quit <-chan struct{}) error { + for { + select { + case <-quit: + return nil + case ev := <-f.input: + // will block the same way as in any feed + output <- ev + } + } + }) +} + +func newFakeServer() *fakePeerEvents { + srv := newFakePeerAdderRemover() + srv.input = make(chan *p2p.PeerEvent, 20) + return srv +} + +type fakeEnvelopeEvents struct { + input chan whisperv6.EnvelopeEvent +} + +func (f fakeEnvelopeEvents) SubscribeEnvelopeEvents(output chan<- whisperv6.EnvelopeEvent) event.Subscription { + return event.NewSubscription(func(quit <-chan struct{}) error { + for { + select { + case <-quit: + return nil + case ev := <-f.input: + // will block the same way as in any feed + output <- ev + } + } + }) +} + +func newFakeEnvelopesEvents() fakeEnvelopeEvents { + return fakeEnvelopeEvents{ + input: make(chan whisperv6.EnvelopeEvent), + } +} + +func getNRandomNodes(t *testing.T, n int) map[enode.ID]*enode.Node { + rst := map[enode.ID]*enode.Node{} + for i := 0; i < n; i++ { + n, err := RandomeNode() + require.NoError(t, err) + rst[n.ID()] = n + } + return rst +} + +func mergeOldIntoNew(old, new map[enode.ID]*enode.Node) { + for n := range old { + new[n] = old[n] + } +} + +func TestReplaceNodes(t *testing.T) { + type testCase struct { + description string + old map[enode.ID]*enode.Node + new map[enode.ID]*enode.Node + target int + } + for _, tc := range []testCase{ + { + "InitialReplace", + getNRandomNodes(t, 0), + getNRandomNodes(t, 3), + 2, + }, + { + "FullReplace", + getNRandomNodes(t, 3), + getNRandomNodes(t, 3), + 2, + }, + } { + t.Run(tc.description, func(t *testing.T) { + peers := newFakePeerAdderRemover() + replaceNodes(peers, tc.target, peers.nodes, nil, tc.old) + require.Len(t, peers.nodes, len(tc.old)) + for n := range peers.nodes { + require.Contains(t, tc.old, n) + } + replaceNodes(peers, tc.target, peers.nodes, tc.old, tc.new) + require.Len(t, peers.nodes, len(tc.new)) + for n := range peers.nodes { + require.Contains(t, tc.new, n) + } + }) + } +} + +func TestPartialReplaceNodesBelowTarget(t *testing.T) { + peers := newFakePeerAdderRemover() + old := getNRandomNodes(t, 1) + new := getNRandomNodes(t, 2) + replaceNodes(peers, 2, peers.nodes, nil, old) + mergeOldIntoNew(old, new) + replaceNodes(peers, 2, peers.nodes, old, new) + require.Len(t, peers.nodes, len(new)) +} + +func TestPartialReplaceNodesAboveTarget(t *testing.T) { + peers := newFakePeerAdderRemover() + old := getNRandomNodes(t, 1) + new := getNRandomNodes(t, 2) + replaceNodes(peers, 1, peers.nodes, nil, old) + mergeOldIntoNew(old, new) + replaceNodes(peers, 1, peers.nodes, old, new) + require.Len(t, peers.nodes, 1) +} + +func TestConnectionManagerAddDrop(t *testing.T) { + server := newFakeServer() + whisper := newFakeEnvelopesEvents() + target := 1 + connmanager := NewConnectionManager(server, whisper, target) + connmanager.Start() + defer connmanager.Stop() + nodes := []*enode.Node{} + for _, n := range getNRandomNodes(t, 3) { + nodes = append(nodes, n) + } + // Send 3 random nodes to connection manager. + connmanager.Notify(nodes) + var initial enode.ID + // Wait till connection manager establishes connection with 1 peer. + require.NoError(t, utils.Eventually(func() error { + nodes := server.Nodes() + if len(nodes) != target { + return fmt.Errorf("unexpected number of connected servers: %d", len(nodes)) + } + initial = nodes[0] + return nil + }, time.Second, 100*time.Millisecond)) + // Send an event that peer was dropped. + select { + case server.input <- &p2p.PeerEvent{Peer: initial, Type: p2p.PeerEventTypeDrop}: + case <-time.After(time.Second): + require.FailNow(t, "can't send a drop event") + } + // Connection manager should establish connection with any other peer from initial list. + require.NoError(t, utils.Eventually(func() error { + nodes := server.Nodes() + if len(nodes) != target { + return fmt.Errorf("unexpected number of connected servers: %d", len(nodes)) + } + if nodes[0] == initial { + return fmt.Errorf("connected node wasn't changed from %s", initial) + } + return nil + }, time.Second, 100*time.Millisecond)) +} + +func TestConnectionManagerReplace(t *testing.T) { + server := newFakeServer() + whisper := newFakeEnvelopesEvents() + target := 1 + connmanager := NewConnectionManager(server, whisper, target) + connmanager.Start() + defer connmanager.Stop() + nodes := []*enode.Node{} + for _, n := range getNRandomNodes(t, 3) { + nodes = append(nodes, n) + } + // Send a single node to connection manager. + connmanager.Notify(nodes[:1]) + // Wait until this node will get connected. + require.NoError(t, utils.Eventually(func() error { + connected := server.Nodes() + if len(connected) != target { + return fmt.Errorf("unexpected number of connected servers: %d", len(connected)) + } + if nodes[0].ID() != connected[0] { + return fmt.Errorf("connected with a wrong peer. expected %s, got %s", nodes[0].ID(), connected[0]) + } + return nil + }, time.Second, 100*time.Millisecond)) + // Replace previously sent node with 2 different nodes. + connmanager.Notify(nodes[1:]) + // Wait until connection manager replaces node connected in the first round. + require.NoError(t, utils.Eventually(func() error { + connected := server.Nodes() + if len(connected) != target { + return fmt.Errorf("unexpected number of connected servers: %d", len(connected)) + } + switch connected[0] { + case nodes[1].ID(): + case nodes[2].ID(): + default: + return fmt.Errorf("connected with unexpected peer. got %s, expected %+v", connected[0], nodes[1:]) + } + return nil + }, time.Second, 100*time.Millisecond)) +} + +func TestConnectionChangedAfterExpiry(t *testing.T) { + server := newFakeServer() + whisper := newFakeEnvelopesEvents() + target := 1 + connmanager := NewConnectionManager(server, whisper, target) + connmanager.Start() + defer connmanager.Stop() + nodes := []*enode.Node{} + for _, n := range getNRandomNodes(t, 2) { + nodes = append(nodes, n) + } + // Send two random nodes to connection manager. + connmanager.Notify(nodes) + var initial enode.ID + // Wait until connection manager establishes connection with one node. + require.NoError(t, utils.Eventually(func() error { + nodes := server.Nodes() + if len(nodes) != target { + return fmt.Errorf("unexpected number of connected servers: %d", len(nodes)) + } + initial = nodes[0] + return nil + }, time.Second, 100*time.Millisecond)) + hash := common.Hash{1} + // Send event that history request for connected peer was sent. + select { + case whisper.input <- whisperv6.EnvelopeEvent{ + Event: whisperv6.EventMailServerRequestSent, Peer: initial, Hash: hash}: + case <-time.After(time.Second): + require.FailNow(t, "can't send a 'sent' event") + } + // And eventually expired. + select { + case whisper.input <- whisperv6.EnvelopeEvent{ + Event: whisperv6.EventMailServerRequestExpired, Peer: initial, Hash: hash}: + case <-time.After(time.Second): + require.FailNow(t, "can't send an 'expiry' event") + } + require.NoError(t, utils.Eventually(func() error { + nodes := server.Nodes() + if len(nodes) != target { + return fmt.Errorf("unexpected number of connected servers: %d", len(nodes)) + } + if nodes[0] == initial { + return fmt.Errorf("connected node wasn't changed from %s", initial) + } + return nil + }, time.Second, 100*time.Millisecond)) +} diff --git a/services/shhext/mailservers/peerstore.go b/services/shhext/mailservers/peerstore.go new file mode 100644 index 00000000000..e545d9f79e3 --- /dev/null +++ b/services/shhext/mailservers/peerstore.go @@ -0,0 +1,67 @@ +package mailservers + +import ( + "errors" + "sync" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +var ( + // ErrNoConnected returned when mail servers are not connected. + ErrNoConnected = errors.New("no connected mail servers") +) + +// PeersProvider is an interface for requesting list of peers. +type PeersProvider interface { + Peers() []*p2p.Peer +} + +// NewPeerStore returns an instance of PeerStore. +func NewPeerStore() *PeerStore { + return &PeerStore{nodes: map[enode.ID]*enode.Node{}} +} + +// PeerStore stores list of selected mail servers and keeps N of them connected. +type PeerStore struct { + mu sync.RWMutex + nodes map[enode.ID]*enode.Node +} + +// Exist confirms that peers was added to a store. +func (ps *PeerStore) Exist(peer enode.ID) bool { + ps.mu.RLock() + defer ps.mu.RUnlock() + _, exist := ps.nodes[peer] + return exist +} + +// Get returns instance of the node with requested ID or nil if ID is not found. +func (ps *PeerStore) Get(peer enode.ID) *enode.Node { + ps.mu.RLock() + defer ps.mu.RUnlock() + return ps.nodes[peer] +} + +// Update updates peers locally. +func (ps *PeerStore) Update(nodes []*enode.Node) { + ps.mu.Lock() + defer ps.mu.Unlock() + ps.nodes = map[enode.ID]*enode.Node{} + for _, n := range nodes { + ps.nodes[n.ID()] = n + } +} + +// GetFirstConnected returns first connected peer that is also added to a peer store. +// Raises ErrNoConnected if no peers are added to a peer store. +func GetFirstConnected(provider PeersProvider, store *PeerStore) (*enode.Node, error) { + peers := provider.Peers() + for _, p := range peers { + if store.Exist(p.ID()) { + return p.Node(), nil + } + } + return nil, ErrNoConnected +} diff --git a/services/shhext/mailservers/peerstore_test.go b/services/shhext/mailservers/peerstore_test.go new file mode 100644 index 00000000000..111b826c306 --- /dev/null +++ b/services/shhext/mailservers/peerstore_test.go @@ -0,0 +1,77 @@ +package mailservers + +import ( + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/stretchr/testify/require" +) + +func RandomeNode() (*enode.Node, error) { + pkey, err := crypto.GenerateKey() + if err != nil { + return nil, err + } + return enode.NewV4(&pkey.PublicKey, nil, 0, 0), nil +} + +func TestUpdateResetsInternalStorage(t *testing.T) { + store := NewPeerStore() + r1, err := RandomeNode() + require.NoError(t, err) + r2, err := RandomeNode() + require.NoError(t, err) + store.Update([]*enode.Node{r1, r2}) + require.True(t, store.Exist(r1.ID())) + require.True(t, store.Exist(r2.ID())) + store.Update([]*enode.Node{r2}) + require.False(t, store.Exist(r1.ID())) + require.True(t, store.Exist(r2.ID())) +} + +func TestGetNodeByID(t *testing.T) { + store := NewPeerStore() + r1, err := RandomeNode() + require.NoError(t, err) + store.Update([]*enode.Node{r1}) + require.Equal(t, r1, store.Get(r1.ID())) + require.Nil(t, store.Get(enode.ID{1})) +} + +type fakePeerProvider struct { + peers []*p2p.Peer +} + +func (f fakePeerProvider) Peers() []*p2p.Peer { + return f.peers +} + +func TestNoConnected(t *testing.T) { + provider := fakePeerProvider{} + store := NewPeerStore() + _, err := GetFirstConnected(provider, store) + require.EqualError(t, ErrNoConnected, err.Error()) +} + +func TestGetFirstConnected(t *testing.T) { + numPeers := 3 + nodes := make([]*enode.Node, numPeers) + peers := make([]*p2p.Peer, numPeers) + nodesMap := getNRandomNodes(t, numPeers) + i := 0 + for _, node := range nodesMap { + nodes[i] = node + peers[i] = p2p.NewPeer(node.ID(), node.ID().String(), nil) + i++ + } + store := NewPeerStore() + provider := fakePeerProvider{peers} + _, err := GetFirstConnected(provider, store) + require.EqualError(t, ErrNoConnected, err.Error()) + store.Update(nodes) + node, err := GetFirstConnected(provider, store) + require.NoError(t, err) + require.Contains(t, nodesMap, node.ID()) +} diff --git a/services/shhext/service.go b/services/shhext/service.go index 59505328427..76421bc6823 100644 --- a/services/shhext/service.go +++ b/services/shhext/service.go @@ -6,35 +6,26 @@ import ( "fmt" "os" "path/filepath" - "sync" - "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rpc" "github.com/status-im/status-go/services/shhext/chat" "github.com/status-im/status-go/services/shhext/dedup" + "github.com/status-im/status-go/services/shhext/mailservers" whisper "github.com/status-im/whisper/whisperv6" "github.com/syndtr/goleveldb/leveldb" ) -var errProtocolNotInitialized = errors.New("procotol is not initialized") - -// EnvelopeState in local tracker -type EnvelopeState int - const ( - // EnvelopePosted is set when envelope was added to a local whisper queue. - EnvelopePosted EnvelopeState = iota - // EnvelopeSent is set when envelope is sent to atleast one peer. - EnvelopeSent - // MailServerRequestSent is set when p2p request is sent to the mailserver - MailServerRequestSent + // defaultConnectionsTarget used in Service.Start if configured connection target is 0. + defaultConnectionsTarget = 1 ) +var errProtocolNotInitialized = errors.New("procotol is not initialized") + // EnvelopeEventsHandler used for two different event types. type EnvelopeEventsHandler interface { EnvelopeSent(common.Hash) @@ -46,7 +37,9 @@ type EnvelopeEventsHandler interface { // Service is a service that provides some additional Whisper API. type Service struct { w *whisper.Whisper + config *ServiceConfig tracker *tracker + server *p2p.Server nodeID *ecdsa.PrivateKey deduplicator *dedup.Deduplicator protocol *chat.ProtocolService @@ -54,6 +47,9 @@ type Service struct { dataDir string installationID string pfsEnabled bool + + peerStore *mailservers.PeerStore + connManager *mailservers.ConnectionManager } type ServiceConfig struct { @@ -62,6 +58,8 @@ type ServiceConfig struct { Debug bool PFSEnabled bool MailServerConfirmations bool + EnableConnectionManager bool + ConnectionTarget int } // Make sure that Service implements node.Service interface. @@ -69,28 +67,34 @@ var _ node.Service = (*Service)(nil) // New returns a new Service. dataDir is a folder path to a network-independent location func New(w *whisper.Whisper, handler EnvelopeEventsHandler, db *leveldb.DB, config *ServiceConfig) *Service { + ps := mailservers.NewPeerStore() track := &tracker{ w: w, handler: handler, cache: map[common.Hash]EnvelopeState{}, batches: map[common.Hash]map[common.Hash]struct{}{}, - mailservers: map[enode.ID]struct{}{}, + mailPeers: ps, mailServerConfirmation: config.MailServerConfirmations, } return &Service{ w: w, + config: config, tracker: track, deduplicator: dedup.NewDeduplicator(w, db), debug: config.Debug, dataDir: config.DataDir, installationID: config.InstallationID, pfsEnabled: config.PFSEnabled, + peerStore: ps, } } // UpdateMailservers updates information about selected mail servers. func (s *Service) UpdateMailservers(nodes []*enode.Node) { - s.tracker.UpdateMailservers(nodes) + s.peerStore.Update(nodes) + if s.connManager != nil { + s.connManager.Notify(nodes) + } } // Protocols returns a new protocols list. In this case, there are none. @@ -191,237 +195,26 @@ func (s *Service) APIs() []rpc.API { // Start is run when a service is started. // It does nothing in this case but is required by `node.Service` interface. func (s *Service) Start(server *p2p.Server) error { + if s.config.EnableConnectionManager { + connectionsTarget := s.config.ConnectionTarget + if connectionsTarget == 0 { + connectionsTarget = defaultConnectionsTarget + } + s.connManager = mailservers.NewConnectionManager(server, s.w, connectionsTarget) + s.connManager.Start() + } s.tracker.Start() s.nodeID = server.PrivateKey + s.server = server return nil } // Stop is run when a service is stopped. // It does nothing in this case but is required by `node.Service` interface. func (s *Service) Stop() error { + if s.config.EnableConnectionManager { + s.connManager.Stop() + } s.tracker.Stop() return nil } - -// tracker responsible for processing events for envelopes that we are interested in -// and calling specified handler. -type tracker struct { - w *whisper.Whisper - handler EnvelopeEventsHandler - mailServerConfirmation bool - - mu sync.Mutex - cache map[common.Hash]EnvelopeState - batches map[common.Hash]map[common.Hash]struct{} - - mailMu sync.Mutex - mailservers map[enode.ID]struct{} - - wg sync.WaitGroup - quit chan struct{} -} - -// Start processing events. -func (t *tracker) Start() { - t.quit = make(chan struct{}) - t.wg.Add(1) - go func() { - t.handleEnvelopeEvents() - t.wg.Done() - }() -} - -// Stop process events. -func (t *tracker) Stop() { - close(t.quit) - t.wg.Wait() -} - -// Add hash to a tracker. -func (t *tracker) Add(hash common.Hash) { - t.mu.Lock() - defer t.mu.Unlock() - t.cache[hash] = EnvelopePosted -} - -func (t *tracker) UpdateMailservers(nodes []*enode.Node) { - t.mailMu.Lock() - defer t.mailMu.Unlock() - t.mailservers = map[enode.ID]struct{}{} - for _, n := range nodes { - t.mailservers[n.ID()] = struct{}{} - } -} - -// Add request hash to a tracker. -func (t *tracker) AddRequest(hash common.Hash, timerC <-chan time.Time) { - t.mu.Lock() - defer t.mu.Unlock() - t.cache[hash] = MailServerRequestSent - go t.expireRequest(hash, timerC) -} - -func (t *tracker) expireRequest(hash common.Hash, timerC <-chan time.Time) { - select { - case <-t.quit: - return - case <-timerC: - t.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventMailServerRequestExpired, - Hash: hash, - }) - } -} - -// handleEnvelopeEvents processes whisper envelope events -func (t *tracker) handleEnvelopeEvents() { - events := make(chan whisper.EnvelopeEvent, 100) // must be buffered to prevent blocking whisper - sub := t.w.SubscribeEnvelopeEvents(events) - defer sub.Unsubscribe() - for { - select { - case <-t.quit: - return - case event := <-events: - t.handleEvent(event) - } - } -} - -// handleEvent based on type of the event either triggers -// confirmation handler or removes hash from tracker -func (t *tracker) handleEvent(event whisper.EnvelopeEvent) { - handlers := map[whisper.EventType]func(whisper.EnvelopeEvent){ - whisper.EventEnvelopeSent: t.handleEventEnvelopeSent, - whisper.EventEnvelopeExpired: t.handleEventEnvelopeExpired, - whisper.EventBatchAcknowledged: t.handleAcknowledgedBatch, - whisper.EventMailServerRequestCompleted: t.handleEventMailServerRequestCompleted, - whisper.EventMailServerRequestExpired: t.handleEventMailServerRequestExpired, - } - - if handler, ok := handlers[event.Event]; ok { - handler(event) - } -} - -func (t *tracker) handleEventEnvelopeSent(event whisper.EnvelopeEvent) { - if t.mailServerConfirmation { - if !t.isMailserver(event.Peer) { - return - } - } - - t.mu.Lock() - defer t.mu.Unlock() - - state, ok := t.cache[event.Hash] - // if we didn't send a message using extension - skip it - // if message was already confirmed - skip it - if !ok || state == EnvelopeSent { - return - } - if event.Batch != (common.Hash{}) { - if _, ok := t.batches[event.Batch]; !ok { - t.batches[event.Batch] = map[common.Hash]struct{}{} - } - t.batches[event.Batch][event.Hash] = struct{}{} - log.Debug("waiting for a confirmation", "batch", event.Batch) - } else { - log.Debug("envelope is sent", "hash", event.Hash, "peer", event.Peer) - t.cache[event.Hash] = EnvelopeSent - if t.handler != nil { - t.handler.EnvelopeSent(event.Hash) - } - } -} - -func (t *tracker) isMailserver(peer enode.ID) bool { - t.mailMu.Lock() - defer t.mailMu.Unlock() - if len(t.mailservers) == 0 { - log.Error("mail servers are empty") - return false - } - _, exist := t.mailservers[peer] - if !exist { - log.Debug("confirmation received not from a mail server is skipped", "peer", peer) - return false - } - return true -} - -func (t *tracker) handleAcknowledgedBatch(event whisper.EnvelopeEvent) { - if t.mailServerConfirmation { - if !t.isMailserver(event.Peer) { - return - } - } - - t.mu.Lock() - defer t.mu.Unlock() - - envelopes, ok := t.batches[event.Batch] - if !ok { - log.Debug("batch is not found", "batch", event.Batch) - } - log.Debug("received a confirmation", "batch", event.Batch, "peer", event.Peer) - for hash := range envelopes { - state, ok := t.cache[hash] - if !ok || state == EnvelopeSent { - continue - } - t.cache[hash] = EnvelopeSent - if t.handler != nil { - t.handler.EnvelopeSent(hash) - } - } - delete(t.batches, event.Batch) -} - -func (t *tracker) handleEventEnvelopeExpired(event whisper.EnvelopeEvent) { - t.mu.Lock() - defer t.mu.Unlock() - - if state, ok := t.cache[event.Hash]; ok { - delete(t.cache, event.Hash) - if state == EnvelopeSent { - return - } - log.Debug("envelope expired", "hash", event.Hash, "state", state) - if t.handler != nil { - t.handler.EnvelopeExpired(event.Hash) - } - } -} - -func (t *tracker) handleEventMailServerRequestCompleted(event whisper.EnvelopeEvent) { - t.mu.Lock() - defer t.mu.Unlock() - - state, ok := t.cache[event.Hash] - if !ok || state != MailServerRequestSent { - return - } - log.Debug("mailserver response received", "hash", event.Hash) - delete(t.cache, event.Hash) - if t.handler != nil { - if resp, ok := event.Data.(*whisper.MailServerResponse); ok { - t.handler.MailServerRequestCompleted(event.Hash, resp.LastEnvelopeHash, resp.Cursor, resp.Error) - } - } -} - -func (t *tracker) handleEventMailServerRequestExpired(event whisper.EnvelopeEvent) { - t.mu.Lock() - defer t.mu.Unlock() - - state, ok := t.cache[event.Hash] - if !ok || state != MailServerRequestSent { - return - } - log.Debug("mailserver response expired", "hash", event.Hash) - delete(t.cache, event.Hash) - if t.handler != nil { - t.handler.MailServerRequestExpired(event.Hash) - } -} diff --git a/services/shhext/service_test.go b/services/shhext/service_test.go index eff42b48019..60216b5fe56 100644 --- a/services/shhext/service_test.go +++ b/services/shhext/service_test.go @@ -95,6 +95,7 @@ func (s *ShhExtSuite) SetupTest() { Debug: true, PFSEnabled: false, MailServerConfirmations: true, + ConnectionTarget: 10, } s.services[i] = New(s.whisper[i], nil, nil, config) s.NoError(stack.Register(func(n *node.ServiceContext) (node.Service, error) { @@ -239,12 +240,12 @@ func (s *ShhExtSuite) TestRequestMessagesSuccess() { }, NoUSB: true, }) // in-memory node as no data dir - s.NoError(err) + s.Require().NoError(err) err = aNode.Register(func(*node.ServiceContext) (node.Service, error) { return shh, nil }) - s.NoError(err) + s.Require().NoError(err) err = aNode.Start() - s.NoError(err) + s.Require().NoError(err) defer func() { err := aNode.Stop(); s.NoError(err) }() mock := newHandlerMock(1) @@ -255,6 +256,7 @@ func (s *ShhExtSuite) TestRequestMessagesSuccess() { PFSEnabled: false, } service := New(shh, mock, nil, config) + s.Require().NoError(service.Start(aNode.Server())) api := NewPublicAPI(service) // with a peer acting as a mailserver @@ -267,41 +269,40 @@ func (s *ShhExtSuite) TestRequestMessagesSuccess() { }, NoUSB: true, }) // in-memory node as no data dir - s.NoError(err) + s.Require().NoError(err) err = mailNode.Register(func(*node.ServiceContext) (node.Service, error) { return whisper.New(nil), nil }) s.NoError(err) err = mailNode.Start() - s.NoError(err) + s.Require().NoError(err) defer func() { s.NoError(mailNode.Stop()) }() // add mailPeer as a peer waitErr := helpers.WaitForPeerAsync(aNode.Server(), mailNode.Server().Self().String(), p2p.PeerEventTypeAdd, time.Second) aNode.Server().AddPeer(mailNode.Server().Self()) - s.NoError(<-waitErr) + s.Require().NoError(<-waitErr) var hash []byte // send a request with a symmetric key symKeyID, symKeyErr := shh.AddSymKeyFromPassword("some-pass") - s.NoError(symKeyErr) + s.Require().NoError(symKeyErr) hash, err = api.RequestMessages(context.TODO(), MessagesRequest{ MailServerPeer: mailNode.Server().Self().String(), SymKeyID: symKeyID, }) - s.NoError(err) - s.NotNil(hash) - s.Contains(api.service.tracker.cache, common.BytesToHash(hash)) - + s.Require().NoError(err) + s.Require().NotNil(hash) + s.Require().NoError(waitForHashInTracker(api.service.tracker, common.BytesToHash(hash), MailServerRequestSent, time.Second)) // Send a request without a symmetric key. In this case, // a public key extracted from MailServerPeer will be used. hash, err = api.RequestMessages(context.TODO(), MessagesRequest{ MailServerPeer: mailNode.Server().Self().String(), }) - s.NoError(err) - s.NotNil(hash) - s.Contains(api.service.tracker.cache, common.BytesToHash(hash)) + s.Require().NoError(err) + s.Require().NotNil(hash) + s.Require().NoError(waitForHashInTracker(api.service.tracker, common.BytesToHash(hash), MailServerRequestSent, time.Second)) } func (s *ShhExtSuite) TestDebugPostSync() { @@ -402,141 +403,17 @@ func (s *ShhExtSuite) TearDown() { } } -var ( - testHash = common.Hash{0x01} -) - -func TestTrackerSuite(t *testing.T) { - suite.Run(t, new(TrackerSuite)) -} - -type TrackerSuite struct { - suite.Suite - - tracker *tracker -} - -func (s *TrackerSuite) SetupTest() { - s.tracker = &tracker{ - cache: map[common.Hash]EnvelopeState{}, - batches: map[common.Hash]map[common.Hash]struct{}{}, - mailservers: map[enode.ID]struct{}{}, - mailServerConfirmation: true, - } -} - -func (s *TrackerSuite) TestConfirmed() { - testPeer := enode.ID{1} - s.tracker.mailservers[testPeer] = struct{}{} - s.tracker.Add(testHash) - s.Contains(s.tracker.cache, testHash) - s.Equal(EnvelopePosted, s.tracker.cache[testHash]) - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventEnvelopeSent, - Hash: testHash, - Peer: testPeer, - }) - s.Contains(s.tracker.cache, testHash) - s.Equal(EnvelopeSent, s.tracker.cache[testHash]) -} - -func (s *TrackerSuite) TestConfirmedWithAcknowledge() { - testBatch := common.Hash{1} - testPeer := enode.ID{1} - s.tracker.Add(testHash) - s.tracker.mailservers[testPeer] = struct{}{} - s.Contains(s.tracker.cache, testHash) - s.Equal(EnvelopePosted, s.tracker.cache[testHash]) - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventEnvelopeSent, - Hash: testHash, - Batch: testBatch, - Peer: testPeer, - }) - s.Equal(EnvelopePosted, s.tracker.cache[testHash]) - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventBatchAcknowledged, - Batch: testBatch, - Peer: testPeer, - }) - s.Contains(s.tracker.cache, testHash) - s.Equal(EnvelopeSent, s.tracker.cache[testHash]) -} - -func (s *TrackerSuite) TestIgnored() { - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventEnvelopeSent, - Hash: testHash, - }) - s.NotContains(s.tracker.cache, testHash) -} - -func (s *TrackerSuite) TestRemoved() { - s.tracker.Add(testHash) - s.Contains(s.tracker.cache, testHash) - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventEnvelopeExpired, - Hash: testHash, - }) - s.NotContains(s.tracker.cache, testHash) -} - -func (s *TrackerSuite) TestRequestCompleted() { - mock := newHandlerMock(1) - s.tracker.handler = mock - s.tracker.AddRequest(testHash, time.After(defaultRequestTimeout*time.Second)) - s.Contains(s.tracker.cache, testHash) - s.Equal(MailServerRequestSent, s.tracker.cache[testHash]) - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventMailServerRequestCompleted, - Hash: testHash, - Data: &whisper.MailServerResponse{}, - }) - select { - case requestID := <-mock.requestsCompleted: - s.Equal(testHash, requestID) - s.NotContains(s.tracker.cache, testHash) - case <-time.After(10 * time.Second): - s.Fail("timed out while waiting for a request to be completed") - } -} - -func (s *TrackerSuite) TestRequestFailed() { - mock := newHandlerMock(1) - s.tracker.handler = mock - s.tracker.AddRequest(testHash, time.After(defaultRequestTimeout*time.Second)) - s.Contains(s.tracker.cache, testHash) - s.Equal(MailServerRequestSent, s.tracker.cache[testHash]) - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventMailServerRequestCompleted, - Hash: testHash, - Data: &whisper.MailServerResponse{Error: errors.New("test error")}, - }) - select { - case requestID := <-mock.requestsFailed: - s.Equal(testHash, requestID) - s.NotContains(s.tracker.cache, testHash) - case <-time.After(10 * time.Second): - s.Fail("timed out while waiting for a request to be failed") - } -} - -func (s *TrackerSuite) TestRequestExpiration() { - mock := newHandlerMock(1) - s.tracker.handler = mock - c := make(chan time.Time) - s.tracker.AddRequest(testHash, c) - s.Contains(s.tracker.cache, testHash) - s.Equal(MailServerRequestSent, s.tracker.cache[testHash]) - s.tracker.handleEvent(whisper.EnvelopeEvent{ - Event: whisper.EventMailServerRequestExpired, - Hash: testHash, - }) - select { - case requestID := <-mock.requestsExpired: - s.Equal(testHash, requestID) - s.NotContains(s.tracker.cache, testHash) - case <-time.After(10 * time.Second): - s.Fail("timed out while waiting for request expiration") +func waitForHashInTracker(track *tracker, hash common.Hash, state EnvelopeState, deadline time.Duration) error { + after := time.After(deadline) + ticker := time.Tick(100 * time.Millisecond) + for { + select { + case <-after: + return fmt.Errorf("failed while waiting for %s to get into state %d", hash, state) + case <-ticker: + if track.GetState(hash) == state { + return nil + } + } } } diff --git a/services/shhext/tracker.go b/services/shhext/tracker.go new file mode 100644 index 00000000000..af1c73bca89 --- /dev/null +++ b/services/shhext/tracker.go @@ -0,0 +1,218 @@ +package shhext + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/status-im/status-go/services/shhext/mailservers" + whisper "github.com/status-im/whisper/whisperv6" +) + +// EnvelopeState in local tracker +type EnvelopeState int + +const ( + // NotRegistered returned if asked hash wasn't registered in the tracker. + NotRegistered EnvelopeState = -1 + // EnvelopePosted is set when envelope was added to a local whisper queue. + EnvelopePosted EnvelopeState = iota + // EnvelopeSent is set when envelope is sent to atleast one peer. + EnvelopeSent + // MailServerRequestSent is set when p2p request is sent to the mailserver + MailServerRequestSent +) + +// tracker responsible for processing events for envelopes that we are interested in +// and calling specified handler. +type tracker struct { + w *whisper.Whisper + handler EnvelopeEventsHandler + mailServerConfirmation bool + + mu sync.Mutex + cache map[common.Hash]EnvelopeState + batches map[common.Hash]map[common.Hash]struct{} + + mailPeers *mailservers.PeerStore + + wg sync.WaitGroup + quit chan struct{} +} + +// Start processing events. +func (t *tracker) Start() { + t.quit = make(chan struct{}) + t.wg.Add(1) + go func() { + t.handleEnvelopeEvents() + t.wg.Done() + }() +} + +// Stop process events. +func (t *tracker) Stop() { + close(t.quit) + t.wg.Wait() +} + +// Add hash to a tracker. +func (t *tracker) Add(hash common.Hash) { + t.mu.Lock() + defer t.mu.Unlock() + t.cache[hash] = EnvelopePosted +} + +func (t *tracker) GetState(hash common.Hash) EnvelopeState { + t.mu.Lock() + defer t.mu.Unlock() + state, exist := t.cache[hash] + if !exist { + return NotRegistered + } + return state +} + +// handleEnvelopeEvents processes whisper envelope events +func (t *tracker) handleEnvelopeEvents() { + events := make(chan whisper.EnvelopeEvent, 100) // must be buffered to prevent blocking whisper + sub := t.w.SubscribeEnvelopeEvents(events) + defer sub.Unsubscribe() + for { + select { + case <-t.quit: + return + case event := <-events: + t.handleEvent(event) + } + } +} + +// handleEvent based on type of the event either triggers +// confirmation handler or removes hash from tracker +func (t *tracker) handleEvent(event whisper.EnvelopeEvent) { + handlers := map[whisper.EventType]func(whisper.EnvelopeEvent){ + whisper.EventEnvelopeSent: t.handleEventEnvelopeSent, + whisper.EventEnvelopeExpired: t.handleEventEnvelopeExpired, + whisper.EventBatchAcknowledged: t.handleAcknowledgedBatch, + whisper.EventMailServerRequestSent: t.handleRequestSent, + whisper.EventMailServerRequestCompleted: t.handleEventMailServerRequestCompleted, + whisper.EventMailServerRequestExpired: t.handleEventMailServerRequestExpired, + } + + if handler, ok := handlers[event.Event]; ok { + handler(event) + } +} + +func (t *tracker) handleEventEnvelopeSent(event whisper.EnvelopeEvent) { + t.mu.Lock() + defer t.mu.Unlock() + + state, ok := t.cache[event.Hash] + // if we didn't send a message using extension - skip it + // if message was already confirmed - skip it + if !ok || state == EnvelopeSent { + return + } + log.Debug("envelope is sent", "hash", event.Hash, "peer", event.Peer) + if event.Batch != (common.Hash{}) { + if _, ok := t.batches[event.Batch]; !ok { + t.batches[event.Batch] = map[common.Hash]struct{}{} + } + t.batches[event.Batch][event.Hash] = struct{}{} + log.Debug("waiting for a confirmation", "batch", event.Batch) + } else { + t.cache[event.Hash] = EnvelopeSent + if t.handler != nil { + t.handler.EnvelopeSent(event.Hash) + } + } +} + +func (t *tracker) isMailserver(peer enode.ID) bool { + return t.mailPeers.Exist(peer) +} + +func (t *tracker) handleAcknowledgedBatch(event whisper.EnvelopeEvent) { + if t.mailServerConfirmation { + if !t.isMailserver(event.Peer) { + return + } + } + + t.mu.Lock() + defer t.mu.Unlock() + + envelopes, ok := t.batches[event.Batch] + if !ok { + log.Debug("batch is not found", "batch", event.Batch) + } + log.Debug("received a confirmation", "batch", event.Batch, "peer", event.Peer) + for hash := range envelopes { + state, ok := t.cache[hash] + if !ok || state == EnvelopeSent { + continue + } + t.cache[hash] = EnvelopeSent + if t.handler != nil { + t.handler.EnvelopeSent(hash) + } + } + delete(t.batches, event.Batch) +} + +func (t *tracker) handleEventEnvelopeExpired(event whisper.EnvelopeEvent) { + t.mu.Lock() + defer t.mu.Unlock() + + if state, ok := t.cache[event.Hash]; ok { + delete(t.cache, event.Hash) + if state == EnvelopeSent { + return + } + log.Debug("envelope expired", "hash", event.Hash, "state", state) + if t.handler != nil { + t.handler.EnvelopeExpired(event.Hash) + } + } +} + +func (t *tracker) handleRequestSent(event whisper.EnvelopeEvent) { + t.mu.Lock() + defer t.mu.Unlock() + t.cache[event.Hash] = MailServerRequestSent +} + +func (t *tracker) handleEventMailServerRequestCompleted(event whisper.EnvelopeEvent) { + t.mu.Lock() + defer t.mu.Unlock() + + state, ok := t.cache[event.Hash] + if !ok || state != MailServerRequestSent { + return + } + log.Debug("mailserver response received", "hash", event.Hash) + delete(t.cache, event.Hash) + if t.handler != nil { + if resp, ok := event.Data.(*whisper.MailServerResponse); ok { + t.handler.MailServerRequestCompleted(event.Hash, resp.LastEnvelopeHash, resp.Cursor, resp.Error) + } + } +} + +func (t *tracker) handleEventMailServerRequestExpired(event whisper.EnvelopeEvent) { + t.mu.Lock() + defer t.mu.Unlock() + + state, ok := t.cache[event.Hash] + if !ok || state != MailServerRequestSent { + return + } + log.Debug("mailserver response expired", "hash", event.Hash) + delete(t.cache, event.Hash) + if t.handler != nil { + t.handler.MailServerRequestExpired(event.Hash) + } +} diff --git a/services/shhext/tracker_test.go b/services/shhext/tracker_test.go new file mode 100644 index 00000000000..718e33f40e5 --- /dev/null +++ b/services/shhext/tracker_test.go @@ -0,0 +1,144 @@ +package shhext + +import ( + "errors" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/status-im/status-go/services/shhext/mailservers" + whisper "github.com/status-im/whisper/whisperv6" + "github.com/stretchr/testify/suite" +) + +var ( + testHash = common.Hash{0x01} +) + +func TestTrackerSuite(t *testing.T) { + suite.Run(t, new(TrackerSuite)) +} + +type TrackerSuite struct { + suite.Suite + + tracker *tracker +} + +func (s *TrackerSuite) SetupTest() { + s.tracker = &tracker{ + cache: map[common.Hash]EnvelopeState{}, + batches: map[common.Hash]map[common.Hash]struct{}{}, + mailPeers: mailservers.NewPeerStore(), + mailServerConfirmation: true, + } +} + +func (s *TrackerSuite) TestConfirmed() { + s.tracker.Add(testHash) + s.Contains(s.tracker.cache, testHash) + s.Equal(EnvelopePosted, s.tracker.cache[testHash]) + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventEnvelopeSent, + Hash: testHash, + }) + s.Contains(s.tracker.cache, testHash) + s.Equal(EnvelopeSent, s.tracker.cache[testHash]) +} + +func (s *TrackerSuite) TestConfirmedWithAcknowledge() { + testBatch := common.Hash{1} + pkey, err := crypto.GenerateKey() + s.Require().NoError(err) + node := enode.NewV4(&pkey.PublicKey, nil, 0, 0) + s.tracker.mailPeers.Update([]*enode.Node{node}) + s.tracker.Add(testHash) + s.Contains(s.tracker.cache, testHash) + s.Equal(EnvelopePosted, s.tracker.cache[testHash]) + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventEnvelopeSent, + Hash: testHash, + Batch: testBatch, + }) + s.Equal(EnvelopePosted, s.tracker.cache[testHash]) + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventBatchAcknowledged, + Batch: testBatch, + Peer: node.ID(), + }) + s.Contains(s.tracker.cache, testHash) + s.Equal(EnvelopeSent, s.tracker.cache[testHash]) +} + +func (s *TrackerSuite) TestIgnored() { + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventEnvelopeSent, + Hash: testHash, + }) + s.NotContains(s.tracker.cache, testHash) +} + +func (s *TrackerSuite) TestRemoved() { + s.tracker.Add(testHash) + s.Contains(s.tracker.cache, testHash) + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventEnvelopeExpired, + Hash: testHash, + }) + s.NotContains(s.tracker.cache, testHash) +} + +func (s *TrackerSuite) TestRequestCompleted() { + mock := newHandlerMock(1) + s.tracker.handler = mock + s.tracker.cache[testHash] = MailServerRequestSent + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventMailServerRequestCompleted, + Hash: testHash, + Data: &whisper.MailServerResponse{}, + }) + select { + case requestID := <-mock.requestsCompleted: + s.Equal(testHash, requestID) + s.NotContains(s.tracker.cache, testHash) + case <-time.After(10 * time.Second): + s.Fail("timed out while waiting for a request to be completed") + } +} + +func (s *TrackerSuite) TestRequestFailed() { + mock := newHandlerMock(1) + s.tracker.handler = mock + s.tracker.cache[testHash] = MailServerRequestSent + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventMailServerRequestCompleted, + Hash: testHash, + Data: &whisper.MailServerResponse{Error: errors.New("test error")}, + }) + select { + case requestID := <-mock.requestsFailed: + s.Equal(testHash, requestID) + s.NotContains(s.tracker.cache, testHash) + case <-time.After(10 * time.Second): + s.Fail("timed out while waiting for a request to be failed") + } +} + +func (s *TrackerSuite) TestRequestExpiration() { + mock := newHandlerMock(1) + s.tracker.handler = mock + s.tracker.cache[testHash] = MailServerRequestSent + s.tracker.handleEvent(whisper.EnvelopeEvent{ + Event: whisper.EventMailServerRequestExpired, + Hash: testHash, + }) + select { + case requestID := <-mock.requestsExpired: + s.Equal(testHash, requestID) + s.NotContains(s.tracker.cache, testHash) + case <-time.After(10 * time.Second): + s.Fail("timed out while waiting for request expiration") + } +} diff --git a/t/e2e/whisper/mailservice_test.go b/t/e2e/whisper/mailservice_test.go index 3a021e35bdb..96ea8df5057 100644 --- a/t/e2e/whisper/mailservice_test.go +++ b/t/e2e/whisper/mailservice_test.go @@ -40,7 +40,9 @@ func (s *MailServiceSuite) TestShhextRequestMessagesRPCMethodAvailability() { // This error means that the method is available through inproc communication // as the validation of params occurred. - err := client.Call(nil, "shhext_requestMessages", map[string]interface{}{}) + err := client.Call(nil, "shhext_requestMessages", map[string]interface{}{ + "mailServerPeer": "xxx", + }) r.EqualError(err, `invalid mailServerPeer value: invalid URL scheme, want "enode"`) // Do the same but using HTTP interface. diff --git a/vendor/github.com/status-im/whisper/whisperv6/events.go b/vendor/github.com/status-im/whisper/whisperv6/events.go index bfbce00c7ff..2faabe1f7d6 100644 --- a/vendor/github.com/status-im/whisper/whisperv6/events.go +++ b/vendor/github.com/status-im/whisper/whisperv6/events.go @@ -17,9 +17,13 @@ const ( EventBatchAcknowledged EventType = "batch.acknowleged" // EventEnvelopeAvailable fires when envelop is available for filters EventEnvelopeAvailable EventType = "envelope.available" + // EventMailServerRequestSent fires when such request is sent. + EventMailServerRequestSent EventType = "mailserver.request.sent" // EventMailServerRequestCompleted fires after mailserver sends all the requested messages EventMailServerRequestCompleted EventType = "mailserver.request.completed" - // EventMailServerRequestExpired fires after mailserver the request TTL ends + // EventMailServerRequestExpired fires after mailserver the request TTL ends. + // This event is independent and concurrent to EventMailServerRequestCompleted. + // Request should be considered as expired only if expiry event was received first. EventMailServerRequestExpired EventType = "mailserver.request.expired" // EventMailServerEnvelopeArchived fires after an envelope has been archived EventMailServerEnvelopeArchived EventType = "mailserver.envelope.archived" diff --git a/vendor/github.com/status-im/whisper/whisperv6/whisper.go b/vendor/github.com/status-im/whisper/whisperv6/whisper.go index cbe44c48aae..c8bed75641a 100644 --- a/vendor/github.com/status-im/whisper/whisperv6/whisper.go +++ b/vendor/github.com/status-im/whisper/whisperv6/whisper.go @@ -407,12 +407,40 @@ func (whisper *Whisper) AllowP2PMessagesFromPeer(peerID []byte) error { // which are not supposed to be forwarded any further. // The whisper protocol is agnostic of the format and contents of envelope. func (whisper *Whisper) RequestHistoricMessages(peerID []byte, envelope *Envelope) error { + return whisper.RequestHistoricMessagesWithTimeout(peerID, envelope, 0) +} + +func (whisper *Whisper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope *Envelope, timeout time.Duration) error { p, err := whisper.getPeer(peerID) if err != nil { return err } + whisper.envelopeFeed.Send(EnvelopeEvent{ + Peer: p.peer.ID(), + Hash: envelope.Hash(), + Event: EventMailServerRequestSent, + }) p.trusted = true - return p2p.Send(p.ws, p2pRequestCode, envelope) + err = p2p.Send(p.ws, p2pRequestCode, envelope) + if timeout != 0 { + go whisper.expireRequestHistoricMessages(p.peer.ID(), envelope.Hash(), timeout) + } + return err +} + +func (whisper *Whisper) expireRequestHistoricMessages(peer enode.ID, hash common.Hash, timeout time.Duration) { + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-whisper.quit: + return + case <-timer.C: + whisper.envelopeFeed.Send(EnvelopeEvent{ + Peer: peer, + Hash: hash, + Event: EventMailServerRequestExpired, + }) + } } func (whisper *Whisper) SendHistoricMessageResponse(peer *Peer, payload []byte) error { From 523d38b55a17e06cbedab8c7433470d9b7913cd2 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Thu, 6 Dec 2018 08:06:06 +0100 Subject: [PATCH 31/61] delete Jenkinsfile-manual --- _assets/ci/Jenkinsfile-manual | 88 ----------------------------------- 1 file changed, 88 deletions(-) delete mode 100644 _assets/ci/Jenkinsfile-manual diff --git a/_assets/ci/Jenkinsfile-manual b/_assets/ci/Jenkinsfile-manual deleted file mode 100644 index e43838fc95f..00000000000 --- a/_assets/ci/Jenkinsfile-manual +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env groovy - -node('linux') { - env.GOPATH = "${env.WORKSPACE}" - env.PATH = "${env.PATH}:${env.GOPATH}/bin" - - cloneDir = 'src/github.com/status-im/status-go' - paramBranch = env.branch ? env.branch : '*/develop' - - dir(cloneDir) { - try { - deleteDir() - } catch (err) { - echo "failure while cleaning the directory" - } - } - - checkout( - changelog: false, - poll: true, - scm: [$class: 'GitSCM', branches: [[name: paramBranch]], - doGenerateSubmoduleConfigurations: false, - extensions: [ - [$class: 'RelativeTargetDirectory', relativeTargetDir: cloneDir] - ], - submoduleCfg: [], - userRemoteConfigs: [[url: 'https://github.com/status-im/status-go']]] - ) - - def remoteOriginRegex = ~/^remotes\/origin\// - - dir(cloneDir) { - gitSHA = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - gitShortSHA = gitSHA.take(8) - gitBranch = sh(returnStdout: true, script: 'git name-rev --name-only HEAD').trim() - remoteOriginRegex - } - - stage('Debug') { - sh 'env' - println(gitBranch) - println(gitSHA) - } - - stage('Test') { - dir(cloneDir) { - sh 'make setup' - sh 'make ci' - } - } - - stage('Build') { - sh 'go get github.com/status-im/xgo' - - parallel ( - 'statusgo-android': { - dir(cloneDir) { - sh 'make statusgo-android' - } - }, - 'statusgo-ios-simulator': { - dir(cloneDir) { - sh ''' - make statusgo-ios-simulator - cd build/bin/statusgo-ios-9.3-framework/ - zip -r status-go-ios.zip Statusgo.framework - ''' - } - } - ) - } - - stage('Deploy') { - dir(cloneDir) { - sh "make prepare-release" - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh """ - yes | make release release_branch=${gitBranch} - """ - } - sh "make clean-release" - } - } -} From a56a3a1913411a6f12fa2c5852bab7ff4c2b92be Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Thu, 3 Jan 2019 18:35:41 +0100 Subject: [PATCH 32/61] add fixes after rebase --- _assets/ci/Jenkinsfile | 23 +++++++++++++++++++---- mobile/status.go | 12 ++++++++++-- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index e4597d7314f..edc545dd8c5 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -107,8 +107,13 @@ pipeline { stage('Prep') { steps { script { lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + version = readFile("${STATUS_PATH}/VERSION").trim() + println("Version: ${version}") println("Git Branch: ${lib.gitBranch()}") println("Git Commit: ${lib.gitCommit()}") + /* save and create a dir for artifacts */ + dest = "${env.WORKSPACE}/pkg" + sh "mkdir -p ${dest}" } } } @@ -139,16 +144,26 @@ pipeline { stage('Cleanup') { steps { script { sh "rm -fr ${dest}" } } } - - stage('Cleanup') { steps { script { - sh "rm -fr ${dest}" - } } } } } // iOS stage('Desktop') { + agent { label 'linux' } + + environment { + STATUS_PATH = 'src/github.com/status-im/status-go' + } + stages { stage('Prepare') { steps { dir(env.STATUS_PATH) { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + version = readFile("${STATUS_PATH}/VERSION").trim() + println("Version: ${version}") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + /* save and create a dir for artifacts */ + dest = "${env.WORKSPACE}/pkg" + sh "mkdir -p ${dest}" sh "zip -r ${dest}/status-go-desktop-${lib.suffix()}.zip . -x *.git" } } } diff --git a/mobile/status.go b/mobile/status.go index 5da19114f49..107aa804c83 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -225,12 +225,20 @@ func ResetChainData() string { // CallRPC calls public APIs via RPC. func CallRPC(inputJSON string) string { - return statusBackend.CallRPC(inputJSON) + resp, err := statusBackend.CallRPC(inputJSON) + if err != nil { + return makeJSONResponse(err) + } + return resp } // CallPrivateRPC calls both public and private APIs via RPC. func CallPrivateRPC(inputJSON string) string { - return statusBackend.CallPrivateRPC(inputJSON) + resp, err := statusBackend.CallPrivateRPC(inputJSON) + if err != nil { + return makeJSONResponse(err) + } + return resp } // CreateAccount is equivalent to creating an account from the command line, From 1524e29759f9986df8705319d4b1eade8cd1f897 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Thu, 3 Jan 2019 18:38:58 +0100 Subject: [PATCH 33/61] fix Desktop stage in Jenkinsfile --- _assets/ci/Jenkinsfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index edc545dd8c5..03bb00ea78d 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -155,17 +155,18 @@ pipeline { } stages { - stage('Prepare') { steps { dir(env.STATUS_PATH) { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - version = readFile("${STATUS_PATH}/VERSION").trim() - println("Version: ${version}") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") - /* save and create a dir for artifacts */ - dest = "${env.WORKSPACE}/pkg" - sh "mkdir -p ${dest}" - sh "zip -r ${dest}/status-go-desktop-${lib.suffix()}.zip . -x *.git" - } } } + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + version = readFile("${STATUS_PATH}/VERSION").trim() + println("Version: ${version}") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + /* save and create a dir for artifacts */ + dest = "${env.WORKSPACE}/pkg" + sh "mkdir -p ${dest}" + } } + } stage('Archive') { steps { archiveArtifacts("pkg/status-go-desktop-${lib.suffix()}.zip") From 57493f465c69e2c1ddcc91c40d922784702d265d Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Thu, 3 Jan 2019 18:44:31 +0100 Subject: [PATCH 34/61] fix Desktop archive in Jenkinsfile --- _assets/ci/Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 03bb00ea78d..7876a204680 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -168,6 +168,10 @@ pipeline { } } } + stage('Zip') { steps { dir(env.STATUS_PATH) { + sh "zip -r ${dest}/status-go-desktop-${lib.suffix()}.zip . -x *.git" + }}} + stage('Archive') { steps { archiveArtifacts("pkg/status-go-desktop-${lib.suffix()}.zip") } } From 6c8ac96763a84836031ba75b57b31fe2643d0723 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Thu, 3 Jan 2019 18:54:52 +0100 Subject: [PATCH 35/61] fix status.go --- mobile/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/status.go b/mobile/status.go index 107aa804c83..54ee3e7bae3 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -444,7 +444,7 @@ func AppStateChange(state string) { // used for gomobile builds func SetMobileSignalHandler(handler SignalHandler) { signal.SetMobileSignalHandler(func(data []byte) { - if data != nil && len(data) > 0 { + if len(data) > 0 { handler.HandleSignal(string(data)) } }) From e6452b4b4b0c44b1d8b551539160d274aa8a4536 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Thu, 24 Jan 2019 16:17:48 +0100 Subject: [PATCH 36/61] fix mobile bindings --- mobile/status.go | 84 +++++++++++++++++++++++++++++++---------------- mobile/types.go | 20 ++++++++--- signal/signals.go | 2 +- 3 files changed, 73 insertions(+), 33 deletions(-) diff --git a/mobile/status.go b/mobile/status.go index 54ee3e7bae3..bf69a4922bf 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -244,7 +244,7 @@ func CallPrivateRPC(inputJSON string) string { // CreateAccount is equivalent to creating an account from the command line, // just modified to handle the function arg passing. func CreateAccount(password string) string { - address, pubKey, mnemonic, err := statusBackend.AccountManager().CreateAccount(password) + info, mnemonic, err := statusBackend.AccountManager().CreateAccount(password) errString := "" if err != nil { @@ -253,10 +253,14 @@ func CreateAccount(password string) string { } out := AccountInfo{ - Address: address, - PubKey: pubKey, - Mnemonic: mnemonic, - Error: errString, + Address: info.WalletAddress, + PubKey: info.WalletPubKey, + WalletAddress: info.WalletAddress, + WalletPubKey: info.WalletPubKey, + ChatAddress: info.ChatAddress, + ChatPubKey: info.ChatPubKey, + Mnemonic: mnemonic, + Error: errString, } outBytes, _ := json.Marshal(out) return string(outBytes) @@ -283,7 +287,7 @@ func CreateChildAccount(parentAddress, password string) string { // RecoverAccount re-creates master key using given details. func RecoverAccount(password, mnemonic string) string { - address, pubKey, err := statusBackend.AccountManager().RecoverAccount(password, mnemonic) + info, err := statusBackend.AccountManager().RecoverAccount(password, mnemonic) errString := "" if err != nil { @@ -292,10 +296,14 @@ func RecoverAccount(password, mnemonic string) string { } out := AccountInfo{ - Address: address, - PubKey: pubKey, - Mnemonic: (mnemonic), - Error: errString, + Address: info.WalletAddress, + PubKey: info.WalletPubKey, + WalletAddress: info.WalletAddress, + WalletPubKey: info.WalletPubKey, + ChatAddress: info.ChatAddress, + ChatPubKey: info.ChatPubKey, + Mnemonic: mnemonic, + Error: errString, } outBytes, _ := json.Marshal(out) return string(outBytes) @@ -311,7 +319,7 @@ func VerifyAccountPassword(keyStoreDir, address, password string) string { // to verify ownership if verified, purges all the previous identities from Whisper, // and injects verified key as shh identity. func Login(address, password string) string { - err := statusBackend.SelectAccount(address, password) + err := statusBackend.SelectAccount(address, address, password) return makeJSONResponse(err) } @@ -393,34 +401,54 @@ func makeJSONResponse(err error) string { return string(outBytes) } -// NotifyUsers sends push notifications by given tokens. -// TODO: remove unusedField -func NotifyUsers(unusedField, payloadJSON, tokensArray string) string { - makeResponse := func(err error) string { - result := NotifyResult{} - - result.Status = err == nil - if err != nil { - result.Error = err.Error() +// SendDataNotification sends push notifications by given tokens. +// dataPayloadJSON is a JSON string that looks like this: +// { +// "data": { +// "msg-v2": { +// "from": "0x2cea3bd5", // hash of sender (first 10 characters/4 bytes of sha3 hash) +// "to": "0xb1f89744", // hash of recipient (first 10 characters/4 bytes of sha3 hash) +// "id": "0x872653ad", // message ID hash (first 10 characters/4 bytes of sha3 hash) +// } +// } +// } +func SendDataNotification(dataPayloadJSON, tokensArray string) (result string) { + var ( + err error + errString string + ) + + defer func() { + out := SendDataNotificationResult{ + Status: err == nil, + Error: errString, } - resultJSON, err := json.Marshal(result) + var resultBytes []byte + resultBytes, err = json.Marshal(out) if err != nil { - logger.Error("failed to marshal Notify output", "error", err) - return makeJSONResponse(err) + logger.Error("failed to marshal SendDataNotification output", "error", err) + result = makeJSONResponse(err) + return } - return string(resultJSON) + result = string(resultBytes) + }() + + tokens, err := ParseJSONArray((tokensArray)) + if err != nil { + errString = err.Error() + return "" } - tokens, err := ParseJSONArray(tokensArray) + err = statusBackend.SendDataNotification(dataPayloadJSON, tokens...) if err != nil { - return makeResponse(err) + errString = err.Error() + return "" } - err = statusBackend.NotifyUsers(payloadJSON, tokens...) - return makeResponse(err) + return "" } // AddPeer adds an enode as a peer. diff --git a/mobile/types.go b/mobile/types.go index 863f5e64214..6504f0d0b64 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -64,10 +64,14 @@ func (e APIError) Error() string { // AccountInfo represents account's info. type AccountInfo struct { - Address string `json:"address"` - PubKey string `json:"pubkey"` - Mnemonic string `json:"mnemonic"` - Error string `json:"error"` + Address string `json:"address"` // DEPRECATED + PubKey string `json:"pubkey"` // DEPRECATED + WalletAddress string `json:"walletAddress"` + WalletPubKey string `json:"walletPubKey"` + ChatAddress string `json:"chatAddress"` + ChatPubKey string `json:"chatPubKey"` + Mnemonic string `json:"mnemonic"` + Error string `json:"error"` } // NotifyResult is a JSON returned from notify message. @@ -76,6 +80,14 @@ type NotifyResult struct { Error string `json:"error,omitempty"` } +// SignalHandler defines a minimal interface +// a signal handler needs to implement. type SignalHandler interface { HandleSignal(string) } + +// SendDataNotificationResult is a JSON returned from notify message. +type SendDataNotificationResult struct { + Status bool `json:"status"` + Error string `json:"error,omitempty"` +} diff --git a/signal/signals.go b/signal/signals.go index 2804df79a2b..f17e9c4d5bd 100644 --- a/signal/signals.go +++ b/signal/signals.go @@ -22,7 +22,7 @@ import ( type MobileSignalHandler func([]byte) // storing the current signal handler here -var mobileSignalHandler MobileSignalHandler = nil +var mobileSignalHandler MobileSignalHandler // All general log messages in this package should be routed through this logger. var logger = log.New("package", "status-go/signal") From f4e64c039c099952f4ae6cccb11f171ec5105016 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Wed, 30 Jan 2019 20:55:12 +0100 Subject: [PATCH 37/61] Use custom NDK for building gomobile. --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index f04b981c7e9..6bec5fa6d69 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,11 @@ release: gomobile-install: go get -u golang.org/x/mobile/cmd/gomobile +ifdef NDK_GOMOBILE + gomobile init -ndk $(NDK_GOMOBILE) +else gomobile init +endif deploy-install: go get -u github.com/c4milo/github-release From 7614f89350bc709ce16118e1ff21ea0a7b99fc0f Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Wed, 30 Jan 2019 21:12:45 +0100 Subject: [PATCH 38/61] (to be reverted) disable canary test --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6bec5fa6d69..87a80580cc6 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,8 @@ test-e2e-race: gotest_extraflags=-race test-e2e-race: test-e2e ##@tests Run e2e tests with -race flag canary-test: node-canary - _assets/scripts/canary_test_mailservers.sh ./config/cli/fleet-eth.beta.json + # TODO: uncomment that! + #_assets/scripts/canary_test_mailservers.sh ./config/cli/fleet-eth.beta.json lint-install: @# The following installs a specific version of golangci-lint, which is appropriate for a CI server to avoid different results from build to build From cd5f884e56f1af01d6bbc8f63c02c7380d8fc70c Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Wed, 30 Jan 2019 21:19:04 +0100 Subject: [PATCH 39/61] (to be reverted) only build mobile release on MacOS --- _assets/ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 7876a204680..7ebdcb5b50c 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { stage('Build') { parallel { stage('Android') { - agent { label 'linux' } + agent { label 'macos' } environment { STATUS_PATH = 'src/github.com/status-im/status-go' From 0468ac384230f1f388d5a0ef3ebfc8371629e119 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Wed, 30 Jan 2019 21:29:48 +0100 Subject: [PATCH 40/61] Revert "(to be reverted) only build mobile release on MacOS" This reverts commit cd5f884e56f1af01d6bbc8f63c02c7380d8fc70c. --- _assets/ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 7ebdcb5b50c..7876a204680 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { stage('Build') { parallel { stage('Android') { - agent { label 'macos' } + agent { label 'linux' } environment { STATUS_PATH = 'src/github.com/status-im/status-go' From ed701372a27891236bd07d3a8f74803b59cd0a2f Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Thu, 31 Jan 2019 14:59:30 +0100 Subject: [PATCH 41/61] fix packages folder conflict in Jenkinsfile --- _assets/ci/Jenkinsfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 7876a204680..37fb8ade83b 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { println("Git Branch: ${lib.gitBranch()}") println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ - dest = "${env.WORKSPACE}/pkg" + dest = "${env.WORKSPACE}/pkg-android" sh "mkdir -p ${dest}" } } } @@ -64,11 +64,11 @@ pipeline { } } } stage('Archive') { steps { - archiveArtifacts("pkg/status-go-android-${lib.suffix()}.aar") + archiveArtifacts("pkg-android/status-go-android-${lib.suffix()}.aar") } } stage('Upload') { steps { script { - lib.uploadArtifact("pkg/status-go-android-${lib.suffix()}.aar") + lib.uploadArtifact("pkg-android/status-go-android-${lib.suffix()}.aar") } } } stage('Release') { when { expression { params.RELEASE == true } } @@ -112,7 +112,7 @@ pipeline { println("Git Branch: ${lib.gitBranch()}") println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ - dest = "${env.WORKSPACE}/pkg" + dest = "${env.WORKSPACE}/pkg-ios" sh "mkdir -p ${dest}" } } } @@ -134,11 +134,11 @@ pipeline { } } } stage('Archive') { steps { - archiveArtifacts("pkg/status-go-ios-${lib.suffix()}.zip") + archiveArtifacts("pkg-ios/status-go-ios-${lib.suffix()}.zip") } } stage('Upload') { steps { script { - lib.uploadArtifact("pkg/status-go-ios-${lib.suffix()}.zip") + lib.uploadArtifact("pkg-ios/status-go-ios-${lib.suffix()}.zip") } } } stage('Cleanup') { steps { script { @@ -163,7 +163,7 @@ pipeline { println("Git Branch: ${lib.gitBranch()}") println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ - dest = "${env.WORKSPACE}/pkg" + dest = "${env.WORKSPACE}/pkg-desktop" sh "mkdir -p ${dest}" } } } @@ -173,11 +173,11 @@ pipeline { }}} stage('Archive') { steps { - archiveArtifacts("pkg/status-go-desktop-${lib.suffix()}.zip") + archiveArtifacts("pkg-desktop/status-go-desktop-${lib.suffix()}.zip") } } stage('Upload') { steps { script { - lib.uploadArtifact("pkg/status-go-desktop-${lib.suffix()}.zip") + lib.uploadArtifact("pkg-desktop/status-go-desktop-${lib.suffix()}.zip") } } } stage('Release') { when { expression { params.RELEASE == true } } From 8f142b9bffc8881ef95987bf817d0121e451c54e Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Thu, 31 Jan 2019 15:49:23 +0100 Subject: [PATCH 42/61] one more attempted fixup for Jenkinsfile --- _assets/ci/Jenkinsfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 37fb8ade83b..9d05812daa3 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -45,8 +45,8 @@ pipeline { println("Git Branch: ${lib.gitBranch()}") println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ - dest = "${env.WORKSPACE}/pkg-android" - sh "mkdir -p ${dest}" + destAndroid = "${env.WORKSPACE}/pkg-android" + sh "mkdir -p ${destAndroid}" } } } @@ -60,7 +60,7 @@ pipeline { stage('Compile') { steps { dir(env.STATUS_PATH) { sh 'make statusgo-android' - sh "cp build/bin/statusgo-android-16.aar ${dest}/status-go-android-${lib.suffix()}.aar" + sh "cp build/bin/statusgo-android-16.aar ${destAndroid}/status-go-android-${lib.suffix()}.aar" } } } stage('Archive') { steps { @@ -89,7 +89,7 @@ pipeline { } stage('Cleanup') { steps { script { - sh "rm -fr ${dest}" + sh "rm -fr ${destAndroid}" } } } } } // Android @@ -112,8 +112,8 @@ pipeline { println("Git Branch: ${lib.gitBranch()}") println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ - dest = "${env.WORKSPACE}/pkg-ios" - sh "mkdir -p ${dest}" + destIOS = "${env.WORKSPACE}/pkg-ios" + sh "mkdir -p ${destIOS}" } } } @@ -129,7 +129,7 @@ pipeline { sh 'make statusgo-ios' dir('build/bin') { sh 'zip -r status-go-ios.zip Statusgo.framework' - sh "cp status-go-ios.zip ${dest}/status-go-ios-${lib.suffix()}.zip" + sh "cp status-go-ios.zip ${destIOS}/status-go-ios-${lib.suffix()}.zip" } } } } @@ -142,7 +142,7 @@ pipeline { } } } stage('Cleanup') { steps { script { - sh "rm -fr ${dest}" + sh "rm -fr ${destIOS}" } } } } } // iOS From e7580e5c2bd8a010f6f7cb8ca6f9fbfa682c73d4 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Thu, 31 Jan 2019 16:07:23 +0100 Subject: [PATCH 43/61] update version (`g` means `gomobile`, update major because backward compat) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index eeb6eac3467..7b9ef2a891c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.20.0-beta.0 +0.21.0g-beta.0 From 12f459b1260362393f8b6cfe200612c127aad108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Thu, 31 Jan 2019 18:41:21 +0100 Subject: [PATCH 44/61] use stash/unstash to release at the end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Sokołowski --- _assets/ci/Jenkinsfile | 98 ++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 9d05812daa3..39f4ac936bc 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent none + agent { label 'linux' } parameters { booleanParam( @@ -20,7 +20,23 @@ pipeline { )) } + environment { + STATUS_PATH = 'src/github.com/status-im/status-go' + GOPATH = "${env.WORKSPACE}" + PATH = "${env.PATH}:${env.GOPATH}/bin" + } + stages { + stage('Prep') { + steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + version = readFile("${STATUS_PATH}/VERSION").trim() + println("Version: ${version}") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + } } + } // stage(Prep) + stage('Build') { parallel { stage('Android') { @@ -39,11 +55,6 @@ pipeline { stages { stage('Prep') { steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - version = readFile("${STATUS_PATH}/VERSION").trim() - println("Version: ${version}") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ destAndroid = "${env.WORKSPACE}/pkg-android" sh "mkdir -p ${destAndroid}" @@ -60,10 +71,11 @@ pipeline { stage('Compile') { steps { dir(env.STATUS_PATH) { sh 'make statusgo-android' - sh "cp build/bin/statusgo-android-16.aar ${destAndroid}/status-go-android-${lib.suffix()}.aar" + sh "cp build/bin/statusgo.aar ${destAndroid}/status-go-android-${lib.suffix()}.aar" } } } stage('Archive') { steps { + stash(name: 'android', includes: "${env.STATUS_PATH}/build/bin/statusgo.aar") archiveArtifacts("pkg-android/status-go-android-${lib.suffix()}.aar") } } @@ -71,28 +83,11 @@ pipeline { lib.uploadArtifact("pkg-android/status-go-android-${lib.suffix()}.aar") } } } - stage('Release') { when { expression { params.RELEASE == true } } - steps { - dir(env.STATUS_PATH) { - sh 'make prepare-release' - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh "yes | make release RELEASE_BRANCH=${lib.gitBranch()}" - } - sh 'make clean-release' - } - } - } - stage('Cleanup') { steps { script { sh "rm -fr ${destAndroid}" } } } } - } // Android + } // stage(Android) stage('iOS') { agent { label 'macos' } @@ -106,11 +101,6 @@ pipeline { stages { stage('Prep') { steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - version = readFile("${STATUS_PATH}/VERSION").trim() - println("Version: ${version}") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ destIOS = "${env.WORKSPACE}/pkg-ios" sh "mkdir -p ${destIOS}" @@ -134,6 +124,7 @@ pipeline { } } } stage('Archive') { steps { + stash(name: 'ios', includes: "${env.STATUS_PATH}/build/bin/status-go-ios.zip") archiveArtifacts("pkg-ios/status-go-ios-${lib.suffix()}.zip") } } @@ -145,7 +136,7 @@ pipeline { sh "rm -fr ${destIOS}" } } } } - } // iOS + } // stage(iOS) stage('Desktop') { agent { label 'linux' } @@ -157,11 +148,6 @@ pipeline { stages { stage('Prep') { steps { script { - lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") - version = readFile("${STATUS_PATH}/VERSION").trim() - println("Version: ${version}") - println("Git Branch: ${lib.gitBranch()}") - println("Git Commit: ${lib.gitCommit()}") /* save and create a dir for artifacts */ dest = "${env.WORKSPACE}/pkg-desktop" sh "mkdir -p ${dest}" @@ -180,29 +166,31 @@ pipeline { lib.uploadArtifact("pkg-desktop/status-go-desktop-${lib.suffix()}.zip") } } } - stage('Release') { when { expression { params.RELEASE == true } } - steps { - dir(env.STATUS_PATH) { - sh 'make prepare-release' - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: 'status-im-auto', - usernameVariable: 'GITHUB_USER', - passwordVariable: 'GITHUB_TOKEN' - ]]) { - sh "yes | make release RELEASE_BRANCH=${lib.gitBranch()}" - } - sh 'make clean-release' - } - } - } - stage('Cleanup') { steps { script { sh "rm -fr ${dest}" } } } } - } // Desktop + } // stage(Desktop) } // parallel - } // stage + } // stage(Build) + + stage('Release') { when { expression { params.RELEASE == true } } + steps { + dir(env.STATUS_PATH) { + unstash 'ios' + unstash 'android' + sh 'make prepare-release' + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: 'status-im-auto', + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ]]) { + sh "yes | make release RELEASE_BRANCH=${lib.gitBranch()}" + } + sh 'make clean-release' + } + } + } // stage(Release) } // stages } From 63ad571730e44425c188d654fc0e28928753339d Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 10:11:07 +0100 Subject: [PATCH 45/61] Fix how we stash Statusgo.framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Sokołowski --- _assets/ci/Jenkinsfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 39f4ac936bc..567b7da16f0 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -65,9 +65,9 @@ pipeline { sh 'make setup-build' } } } - stage('Lint') { steps { dir(env.STATUS_PATH) { - sh 'make ci' - } } } + //stage('Lint') { steps { dir(env.STATUS_PATH) { + // sh 'make ci' + //} } } stage('Compile') { steps { dir(env.STATUS_PATH) { sh 'make statusgo-android' @@ -111,9 +111,9 @@ pipeline { sh 'make setup-build' } } } - stage('Lint') { steps { dir(env.STATUS_PATH) { - sh 'make ci' - } } } + //stage('Lint') { steps { dir(env.STATUS_PATH) { + // sh 'make ci' + //} } } stage('Compile') { steps { dir(env.STATUS_PATH) { sh 'make statusgo-ios' @@ -124,7 +124,9 @@ pipeline { } } } stage('Archive') { steps { - stash(name: 'ios', includes: "${env.STATUS_PATH}/build/bin/status-go-ios.zip") + dir("${env.STATUS_PATH}/build/bin/Statusgo.framework") { + stash(name: 'ios', includes: '**') + } archiveArtifacts("pkg-ios/status-go-ios-${lib.suffix()}.zip") } } From 44e373ad918714fef07cf85d7489c1c919a354f9 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 11:19:33 +0100 Subject: [PATCH 46/61] Tweak "Release" phase. --- _assets/ci/Jenkinsfile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 567b7da16f0..2bec54ad59c 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -77,6 +77,7 @@ pipeline { stage('Archive') { steps { stash(name: 'android', includes: "${env.STATUS_PATH}/build/bin/statusgo.aar") archiveArtifacts("pkg-android/status-go-android-${lib.suffix()}.aar") + stash(name: 'android', includes: "${env.STATUS_PATH}/build/bin/statusgo.aar") } } stage('Upload') { steps { script { @@ -124,8 +125,8 @@ pipeline { } } } stage('Archive') { steps { - dir("${env.STATUS_PATH}/build/bin/Statusgo.framework") { - stash(name: 'ios', includes: '**') + dir("${env.STATUS_PATH}/build/bin") { + stash(name: 'ios', includes: 'status-go-ios.zip') } archiveArtifacts("pkg-ios/status-go-ios-${lib.suffix()}.zip") } } @@ -178,9 +179,14 @@ pipeline { stage('Release') { when { expression { params.RELEASE == true } } steps { - dir(env.STATUS_PATH) { - unstash 'ios' + outputDir = "${env.STATUS_PATH}/build/bin" + sh "mkdir -p ${outputDir}" + dir(outputDir) { + unstash 'ios' // zip file + sh 'unzip status-go-ios.zip' unstash 'android' + } + dir(env.STATUS_PATH) { sh 'make prepare-release' withCredentials([[ $class: 'UsernamePasswordMultiBinding', From 09c9025f20b4d9262d0c46a1b8ae5669de362f3e Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 11:24:08 +0100 Subject: [PATCH 47/61] fix syntax error in Jenkinsfile (eeh) --- _assets/ci/Jenkinsfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 2bec54ad59c..24810b49010 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -179,14 +179,16 @@ pipeline { stage('Release') { when { expression { params.RELEASE == true } } steps { - outputDir = "${env.STATUS_PATH}/build/bin" - sh "mkdir -p ${outputDir}" - dir(outputDir) { - unstash 'ios' // zip file - sh 'unzip status-go-ios.zip' - unstash 'android' - } dir(env.STATUS_PATH) { + outputDir = "${env.STATUS_PATH}/build/bin" + sh "mkdir -p ${outputDir}" + + dir(outputDir) { + unstash 'ios' // zip file + sh 'unzip status-go-ios.zip' + unstash 'android' + } + sh 'make prepare-release' withCredentials([[ $class: 'UsernamePasswordMultiBinding', From 3553a48046173078d5a6ddf2f6e35fbb30fe337f Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 11:25:09 +0100 Subject: [PATCH 48/61] one more syntax fixup --- _assets/ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 24810b49010..82dc30f51f6 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -180,7 +180,7 @@ pipeline { stage('Release') { when { expression { params.RELEASE == true } } steps { dir(env.STATUS_PATH) { - outputDir = "${env.STATUS_PATH}/build/bin" + def outputDir = "${env.STATUS_PATH}/build/bin" sh "mkdir -p ${outputDir}" dir(outputDir) { From 0e0f7f48a4d0b2012286cc8a8c1296eedea85c56 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 11:26:24 +0100 Subject: [PATCH 49/61] :( --- _assets/ci/Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 82dc30f51f6..31578204104 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -180,10 +180,9 @@ pipeline { stage('Release') { when { expression { params.RELEASE == true } } steps { dir(env.STATUS_PATH) { - def outputDir = "${env.STATUS_PATH}/build/bin" - sh "mkdir -p ${outputDir}" + sh "mkdir -p ${env.STATUS_PATH}/build/bin" - dir(outputDir) { + dir("${env.STATUS_PATH}/build/bin") { unstash 'ios' // zip file sh 'unzip status-go-ios.zip' unstash 'android' From bd324000b7fc07aa780adf8397e816dbdf3ec21c Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 11:44:44 +0100 Subject: [PATCH 50/61] add debug output --- _assets/ci/Jenkinsfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 31578204104..3d5ff149b2d 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -185,9 +185,15 @@ pipeline { dir("${env.STATUS_PATH}/build/bin") { unstash 'ios' // zip file sh 'unzip status-go-ios.zip' + sh 'pwd' + sh 'ls' unstash 'android' } + sh 'pwd' + sh 'ls' + sh 'ls build/bin' + sh 'make prepare-release' withCredentials([[ $class: 'UsernamePasswordMultiBinding', From 0e2aeba5fbc1218ade6693d147bf98fb2648432b Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 11:52:29 +0100 Subject: [PATCH 51/61] add cleanup code --- _assets/ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 3d5ff149b2d..afbbe3556ed 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -184,7 +184,7 @@ pipeline { dir("${env.STATUS_PATH}/build/bin") { unstash 'ios' // zip file - sh 'unzip status-go-ios.zip' + sh 'yes | unzip status-go-ios.zip' sh 'pwd' sh 'ls' unstash 'android' From 820e3980fce88d862e0aef353bff6df2df0d9d9e Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 12:00:22 +0100 Subject: [PATCH 52/61] fix dir path --- _assets/ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index afbbe3556ed..d59f5cf0f9e 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -182,7 +182,7 @@ pipeline { dir(env.STATUS_PATH) { sh "mkdir -p ${env.STATUS_PATH}/build/bin" - dir("${env.STATUS_PATH}/build/bin") { + dir("build/bin") { unstash 'ios' // zip file sh 'yes | unzip status-go-ios.zip' sh 'pwd' From c787063b853b0a80a64b92fa03bb2f5dc756df1a Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 13:24:53 +0100 Subject: [PATCH 53/61] make Android stash look more like iOS --- _assets/ci/Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index d59f5cf0f9e..f6d26c07d9c 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -75,9 +75,10 @@ pipeline { } } } stage('Archive') { steps { - stash(name: 'android', includes: "${env.STATUS_PATH}/build/bin/statusgo.aar") + dir("${env.STATUS_PATH}/build/bin") { + stash(name: 'ios', includes: 'statusgo.aar') + } archiveArtifacts("pkg-android/status-go-android-${lib.suffix()}.aar") - stash(name: 'android', includes: "${env.STATUS_PATH}/build/bin/statusgo.aar") } } stage('Upload') { steps { script { From bd8285d97af52e4a6b9256907c3d3b75567e92a6 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 13:35:46 +0100 Subject: [PATCH 54/61] Clean-up after yourself. --- _assets/ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index f6d26c07d9c..a361d2ca024 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -205,6 +205,7 @@ pipeline { sh "yes | make release RELEASE_BRANCH=${lib.gitBranch()}" } sh 'make clean-release' + sh "rm -fr ${env.STATUS_PATH}/build/bin" } } } // stage(Release) From ae680183547a8976ad3391693033198b224d94ba Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 13:47:07 +0100 Subject: [PATCH 55/61] more debug output --- _assets/ci/Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index a361d2ca024..454b41c90d1 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -185,10 +185,14 @@ pipeline { dir("build/bin") { unstash 'ios' // zip file - sh 'yes | unzip status-go-ios.zip' sh 'pwd' sh 'ls' + + sh 'yes | unzip status-go-ios.zip' + unstash 'android' + sh 'pwd' + sh 'ls' } sh 'pwd' From a0141f626e05f6f42c633b07f895fc0ba70a6d51 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 14:11:40 +0100 Subject: [PATCH 56/61] even more debug info --- _assets/ci/Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 454b41c90d1..184d5044dea 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -184,6 +184,9 @@ pipeline { sh "mkdir -p ${env.STATUS_PATH}/build/bin" dir("build/bin") { + sh 'pwd' + sh 'ls' + unstash 'ios' // zip file sh 'pwd' sh 'ls' From e1fb48ead29a700592a249979f72d882cabe7726 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 14:47:13 +0100 Subject: [PATCH 57/61] add more debug info --- _assets/ci/Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 184d5044dea..aca78e3d8a5 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -127,6 +127,8 @@ pipeline { stage('Archive') { steps { dir("${env.STATUS_PATH}/build/bin") { + sh 'pwd' + sh 'ls' stash(name: 'ios', includes: 'status-go-ios.zip') } archiveArtifacts("pkg-ios/status-go-ios-${lib.suffix()}.zip") @@ -187,13 +189,13 @@ pipeline { sh 'pwd' sh 'ls' - unstash 'ios' // zip file + unstash(name: 'ios') // zip file sh 'pwd' sh 'ls' sh 'yes | unzip status-go-ios.zip' - unstash 'android' + unstash(name: 'android') sh 'pwd' sh 'ls' } From a88795b88d774b93a484d8a2b3d58e084d0527e8 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 14:56:15 +0100 Subject: [PATCH 58/61] try not to clean-up files after stashing --- _assets/ci/Jenkinsfile | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index aca78e3d8a5..41ef15504fd 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -84,10 +84,6 @@ pipeline { stage('Upload') { steps { script { lib.uploadArtifact("pkg-android/status-go-android-${lib.suffix()}.aar") } } } - - stage('Cleanup') { steps { script { - sh "rm -fr ${destAndroid}" - } } } } } // stage(Android) @@ -137,10 +133,6 @@ pipeline { stage('Upload') { steps { script { lib.uploadArtifact("pkg-ios/status-go-ios-${lib.suffix()}.zip") } } } - - stage('Cleanup') { steps { script { - sh "rm -fr ${destIOS}" - } } } } } // stage(iOS) @@ -171,10 +163,6 @@ pipeline { stage('Upload') { steps { script { lib.uploadArtifact("pkg-desktop/status-go-desktop-${lib.suffix()}.zip") } } } - - stage('Cleanup') { steps { script { - sh "rm -fr ${dest}" - } } } } } // stage(Desktop) } // parallel From 4a5ef3a51eb190e0acda3792cedceb753cdad8f4 Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 15:08:35 +0100 Subject: [PATCH 59/61] Add `UpdateMailservers` to gomobile exports. --- mobile/status.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mobile/status.go b/mobile/status.go index bf69a4922bf..6d0a6e3233b 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -451,6 +451,18 @@ func SendDataNotification(dataPayloadJSON, tokensArray string) (result string) { return "" } +// UpdateMailservers updates mail servers in status backend. +//export UpdateMailservers +func UpdateMailservers(data string) string { + var enodes []string + err := json.Unmarshal([]byte(data), &enodes) + if err != nil { + return makeJSONResponse(err) + } + err = statusBackend.UpdateMailservers(enodes) + return makeJSONResponse(err) +} + // AddPeer adds an enode as a peer. func AddPeer(enode string) string { err := statusBackend.StatusNode().AddPeer(enode) From 1fa9103113f7f247ea53cd141bb93d1b9a427c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Fri, 1 Feb 2019 15:37:22 +0100 Subject: [PATCH 60/61] split builds into separate Jenkinsfiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Sokołowski --- Makefile | 4 +- _assets/ci/Jenkinsfile | 197 ++++++++------------------------- _assets/ci/Jenkinsfile.android | 65 +++++++++++ _assets/ci/Jenkinsfile.ios | 64 +++++++++++ _assets/ci/Jenkinsfile.linux | 46 ++++++++ _assets/ci/lib.groovy | 33 ++++++ 6 files changed, 256 insertions(+), 153 deletions(-) create mode 100644 _assets/ci/Jenkinsfile.android create mode 100644 _assets/ci/Jenkinsfile.ios create mode 100644 _assets/ci/Jenkinsfile.linux diff --git a/Makefile b/Makefile index 87a80580cc6..ab66eacec66 100644 --- a/Makefile +++ b/Makefile @@ -187,7 +187,7 @@ install-os-dependencies: setup-dev: setup-build install-os-dependencies gen-install update-fleet-config ##@other Prepare project for development -setup-build: dep-install lint-install mock-install deploy-install gomobile-install ##@other Prepare project for build +setup-build: dep-install lint-install mock-install release-install gomobile-install ##@other Prepare project for build setup: setup-build setup-dev ##@other Prepare project for development and building @@ -226,7 +226,7 @@ else gomobile init endif -deploy-install: +release-install: go get -u github.com/c4milo/github-release gen-install: diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index 41ef15504fd..1d39a5b7d5f 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -24,6 +24,8 @@ pipeline { STATUS_PATH = 'src/github.com/status-im/status-go' GOPATH = "${env.WORKSPACE}" PATH = "${env.PATH}:${env.GOPATH}/bin" + /* This will override the var in Makefile */ + RELEASE_DIRECTORY = "${env.WORKSPACE}/pkg" } stages { @@ -37,174 +39,67 @@ pipeline { } } } // stage(Prep) + stage('Setup') { steps { dir(env.STATUS_PATH) { + /* install release tools */ + sh 'make release-install' + } } } // stage(Setup) + stage('Build') { parallel { - stage('Android') { - agent { label 'linux' } - - environment { - STATUS_PATH = 'src/github.com/status-im/status-go' - GOPATH = "${env.WORKSPACE}" - PATH = "${env.PATH}:${env.GOPATH}/bin" - ANDROID_HOME = '/usr/lib/android-sdk' - ANDROID_SDK_ROOT = '/usr/lib/android-sdk' - ANDROID_NDK = '/usr/lib/android-ndk' - ANDROID_NDK_HOME = '/usr/lib/android-ndk' - } - - stages { - stage('Prep') { - steps { script { - /* save and create a dir for artifacts */ - destAndroid = "${env.WORKSPACE}/pkg-android" - sh "mkdir -p ${destAndroid}" - } } - } - - stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup-build' - } } } - - //stage('Lint') { steps { dir(env.STATUS_PATH) { - // sh 'make ci' - //} } } - - stage('Compile') { steps { dir(env.STATUS_PATH) { - sh 'make statusgo-android' - sh "cp build/bin/statusgo.aar ${destAndroid}/status-go-android-${lib.suffix()}.aar" - } } } - - stage('Archive') { steps { - dir("${env.STATUS_PATH}/build/bin") { - stash(name: 'ios', includes: 'statusgo.aar') - } - archiveArtifacts("pkg-android/status-go-android-${lib.suffix()}.aar") - } } - - stage('Upload') { steps { script { - lib.uploadArtifact("pkg-android/status-go-android-${lib.suffix()}.aar") - } } } - } - } // stage(Android) - - stage('iOS') { - agent { label 'macos' } - - environment { - STATUS_PATH = 'src/github.com/status-im/status-go' - GOPATH = "${env.WORKSPACE}" - PATH = "${env.PATH}:${env.GOPATH}/bin" - } - - stages { - stage('Prep') { - steps { script { - /* save and create a dir for artifacts */ - destIOS = "${env.WORKSPACE}/pkg-ios" - sh "mkdir -p ${destIOS}" - } } - } - - stage('Setup') { steps { dir(env.STATUS_PATH) { - sh 'make setup-build' - } } } - - //stage('Lint') { steps { dir(env.STATUS_PATH) { - // sh 'make ci' - //} } } - - stage('Compile') { steps { dir(env.STATUS_PATH) { - sh 'make statusgo-ios' - dir('build/bin') { - sh 'zip -r status-go-ios.zip Statusgo.framework' - sh "cp status-go-ios.zip ${destIOS}/status-go-ios-${lib.suffix()}.zip" - } - } } } - - stage('Archive') { steps { - dir("${env.STATUS_PATH}/build/bin") { - sh 'pwd' - sh 'ls' - stash(name: 'ios', includes: 'status-go-ios.zip') - } - archiveArtifacts("pkg-ios/status-go-ios-${lib.suffix()}.zip") - } } - - stage('Upload') { steps { script { - lib.uploadArtifact("pkg-ios/status-go-ios-${lib.suffix()}.zip") - } } } - } - } // stage(iOS) - - stage('Desktop') { - agent { label 'linux' } - - environment { - STATUS_PATH = 'src/github.com/status-im/status-go' - } - - stages { - stage('Prep') { - steps { script { - /* save and create a dir for artifacts */ - dest = "${env.WORKSPACE}/pkg-desktop" - sh "mkdir -p ${dest}" - } } - } - - stage('Zip') { steps { dir(env.STATUS_PATH) { - sh "zip -r ${dest}/status-go-desktop-${lib.suffix()}.zip . -x *.git" - }}} - - stage('Archive') { steps { - archiveArtifacts("pkg-desktop/status-go-desktop-${lib.suffix()}.zip") - } } - - stage('Upload') { steps { script { - lib.uploadArtifact("pkg-desktop/status-go-desktop-${lib.suffix()}.zip") - } } } - } - } // stage(Desktop) + stage('iOS') { steps { script { + ios = lib.buildBranch('status-go/platforms/ios') + } } } + stage('Android') { steps { script { + android = lib.buildBranch('status-go/platforms/android') + } } } + stage('Linux') { steps { script { + linux = lib.buildBranch('status-go/platforms/linux') + } } } } // parallel } // stage(Build) + stage('Archive') { + steps { script { + sh("rm -f ${env.RELEASE_DIRECTORY}/*") + lib.copyArts('status-go/platforms/ios', ios.number) + lib.copyArts('status-go/platforms/android', android.number) + lib.copyArts('status-go/platforms/linux', linux.number) + dir(env.RELEASE_DIRECTORY) { + /* generate sha256 checksums for upload */ + sh 'sha256sum * | tee checksum.sha256' + archiveArtifacts('*') + } + } } + } // stage(Archive) + stage('Release') { when { expression { params.RELEASE == true } } steps { + /* rename build files to not include versions */ + dir(env.RELEASE_DIRECTORY) { + sh 'mv status-go-ios-*.zip status-go-ios.zip' + sh 'mv status-go-android-*.aar status-go-android.aar' + sh 'mv status-go-desktop-*.zip status-go-desktop.zip' + } + /* perform the release */ dir(env.STATUS_PATH) { - sh "mkdir -p ${env.STATUS_PATH}/build/bin" - - dir("build/bin") { - sh 'pwd' - sh 'ls' - - unstash(name: 'ios') // zip file - sh 'pwd' - sh 'ls' - - sh 'yes | unzip status-go-ios.zip' - - unstash(name: 'android') - sh 'pwd' - sh 'ls' - } - - sh 'pwd' - sh 'ls' - sh 'ls build/bin' - - sh 'make prepare-release' withCredentials([[ $class: 'UsernamePasswordMultiBinding', credentialsId: 'status-im-auto', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN' ]]) { - sh "yes | make release RELEASE_BRANCH=${lib.gitBranch()}" + sh """ + yes | make release \ + RELEASE_BRANCH=${lib.gitBranch()} \ + RELEASE_DIRECTORY=${env.RELEASE_DIRECTORY} + """ } - sh 'make clean-release' - sh "rm -fr ${env.STATUS_PATH}/build/bin" } } } // stage(Release) + + stage('Cleanup') { steps { dir(env.STATUS_PATH) { + sh 'make clean-release' + } } } // stage(Cleanup) } // stages } diff --git a/_assets/ci/Jenkinsfile.android b/_assets/ci/Jenkinsfile.android new file mode 100644 index 00000000000..59944ca3a1d --- /dev/null +++ b/_assets/ci/Jenkinsfile.android @@ -0,0 +1,65 @@ +pipeline { + agent { label 'linux' } + + options { + /* Go requires a certain directory structure */ + checkoutToSubdirectory('src/github.com/status-im/status-go') + /* manage how many builds we keep */ + buildDiscarder(logRotator( + numToKeepStr: '30', + daysToKeepStr: '30', + )) + } + + environment { + STATUS_PATH = 'src/github.com/status-im/status-go' + CI_DIR = "${env.STATUS_PATH}/_assets/ci" + GOPATH = "${env.WORKSPACE}" + PATH = "${env.PATH}:${env.GOPATH}/bin" + ANDROID_HOME = '/usr/lib/android-sdk' + ANDROID_SDK_ROOT = '/usr/lib/android-sdk' + ANDROID_NDK = '/usr/lib/android-ndk' + ANDROID_NDK_HOME = '/usr/lib/android-ndk' + } + + stages { + stage('Prep') { steps { script { + lib = load("${CI_DIR}/lib.groovy") + /* clarify what we're building */ + println("Version: ${lib.getVersion()}") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + /* save and create a dir for artifacts */ + dest = "${env.WORKSPACE}/pkg" + sh "mkdir -p ${dest}" + /* for easier reuse */ + artifact = "status-go-android-${lib.suffix()}.aar" + } } } + + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup-build' + } } } + + stage('Test') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } + + stage('Compile') { steps { dir(env.STATUS_PATH) { + sh 'make statusgo-android' + sh "cp build/bin/statusgo.aar ${dest}/${artifact}" + } } } + + stage('Archive') { steps { + archiveArtifacts("pkg/${artifact}") + } } + + stage('Upload') { steps { script { + lib.uploadArtifact("pkg/${artifact}") + } } } + + stage('Cleanup') { steps { dir(env.STATUS_PATH) { + sh 'make clean' + sh "rm -fr ${dest}" + } } } + } // stages +} // pipeline diff --git a/_assets/ci/Jenkinsfile.ios b/_assets/ci/Jenkinsfile.ios new file mode 100644 index 00000000000..d479d6d9286 --- /dev/null +++ b/_assets/ci/Jenkinsfile.ios @@ -0,0 +1,64 @@ +pipeline { + agent { label 'macos' } + + options { + /* Go requires a certain directory structure */ + checkoutToSubdirectory('src/github.com/status-im/status-go') + /* manage how many builds we keep */ + buildDiscarder(logRotator( + numToKeepStr: '30', + daysToKeepStr: '30', + )) + } + + environment { + STATUS_PATH = 'src/github.com/status-im/status-go' + CI_DIR = "${env.STATUS_PATH}/_assets/ci" + GOPATH = "${env.WORKSPACE}" + PATH = "${env.PATH}:${env.GOPATH}/bin" + } + + stages { + stage('Prep') { steps { script { + lib = load("${CI_DIR}/lib.groovy") + /* clarify what we're building */ + println("Version: ${lib.getVersion()}") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + /* save and create a dir for artifacts */ + dest = "${env.WORKSPACE}/pkg" + sh "mkdir -p ${dest}" + /* for easier reuse */ + artifact = "status-go-ios-${lib.suffix()}.zip" + } } } + + stage('Setup') { steps { dir(env.STATUS_PATH) { + sh 'make setup-build' + } } } + + stage('Test') { steps { dir(env.STATUS_PATH) { + sh 'make ci' + } } } + + stage('Compile') { steps { dir(env.STATUS_PATH) { + sh 'make statusgo-ios' + dir('build/bin') { + sh 'zip -r status-go-ios.zip Statusgo.framework' + sh "cp status-go-ios.zip ${dest}/${artifact}" + } + } } } + + stage('Archive') { steps { + archiveArtifacts("pkg/${artifact}") + } } + + stage('Upload') { steps { script { + lib.uploadArtifact("pkg/${artifact}") + } } } + + stage('Cleanup') { steps { dir(env.STATUS_PATH) { + sh 'make clean' + sh "rm -fr ${dest}" + } } } + } // stages +} // pipeline diff --git a/_assets/ci/Jenkinsfile.linux b/_assets/ci/Jenkinsfile.linux new file mode 100644 index 00000000000..2d172f72743 --- /dev/null +++ b/_assets/ci/Jenkinsfile.linux @@ -0,0 +1,46 @@ +pipeline { + agent { label 'linux' } + + options { + /* Go requires a certain directory structure */ + checkoutToSubdirectory('src/github.com/status-im/status-go') + /* manage how many builds we keep */ + buildDiscarder(logRotator( + numToKeepStr: '30', + daysToKeepStr: '30', + )) + } + + environment { + STATUS_PATH = 'src/github.com/status-im/status-go' + GOPATH = "${env.WORKSPACE}" + PATH = "${env.PATH}:${env.GOPATH}/bin" + } + + stages { + stage('Prep') { steps { script { + lib = load("${env.STATUS_PATH}/_assets/ci/lib.groovy") + /* clarify what we're building */ + println("Version: ${lib.getVersion()}") + println("Git Branch: ${lib.gitBranch()}") + println("Git Commit: ${lib.gitCommit()}") + /* save and create a dir for artifacts */ + dest = "${env.WORKSPACE}/pkg" + sh "mkdir -p ${dest}" + /* for easier reuse */ + artifact = "status-go-desktop-${lib.suffix()}.zip" + } } } + + stage('Compress') { steps { dir(env.STATUS_PATH) { + sh "zip -q -r ${dest}/${artifact} . -x *.git" + } } } + + stage('Archive') { steps { + archiveArtifacts("pkg/${artifact}") + } } + + stage('Upload') { steps { script { + lib.uploadArtifact("pkg/${artifact}") + } } } + } // stages +} // pipeline diff --git a/_assets/ci/lib.groovy b/_assets/ci/lib.groovy index 21b9ddb9ba5..63a80b4f74f 100644 --- a/_assets/ci/lib.groovy +++ b/_assets/ci/lib.groovy @@ -1,3 +1,7 @@ +def getVersion() { + return readFile("${env.STATUS_PATH}/VERSION").trim() +} + def gitCommit() { return GIT_COMMIT.take(6) } @@ -49,4 +53,33 @@ def uploadArtifact(path) { return "https://${bucket}.${domain}/${getFilename(path)}" } +def buildBranch(name = null, buildType = null) { + /* need to drop origin/ to match definitions of child jobs */ + def branchName = env.GIT_BRANCH.replace('origin/', '') + /* always pass the BRANCH and BUILD_TYPE params with current branch */ + def resp = build( + job: name, + /* this allows us to analize the job even after failure */ + propagate: false, + parameters: [ + [name: 'BRANCH', value: branchName, $class: 'StringParameterValue'], + ]) + /* BlueOcean seems to not show child-build links */ + println "Build: ${resp.getAbsoluteUrl()} (${resp.result})" + if (resp.result != 'SUCCESS') { + error("Build Failed") + } + return resp +} + +def copyArts(projectName, buildNo) { + copyArtifacts( + projectName: projectName, + target: 'pkg', + flatten: true, + selector: specific("${buildNo}") + ) +} + return this + From 47557cdbe3c449637cc11683461691f49a204b3e Mon Sep 17 00:00:00 2001 From: Igor Mandrigin Date: Fri, 1 Feb 2019 17:17:44 +0100 Subject: [PATCH 61/61] bump version to .1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7b9ef2a891c..1351453f98d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.21.0g-beta.0 +0.21.0g-beta.1