diff --git a/_assets/build/Dockerfile b/_assets/build/Dockerfile index aff851e799..e96cfcf073 100644 --- a/_assets/build/Dockerfile +++ b/_assets/build/Dockerfile @@ -1,5 +1,5 @@ # Build status-go in a Go builder container -FROM golang:1.12.5-alpine as builder +FROM golang:1.13-alpine as builder RUN apk add --no-cache make gcc musl-dev linux-headers diff --git a/go.mod b/go.mod index 581a58b627..ec882e08e0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/status-im/status-go -go 1.12 +go 1.13 replace github.com/ethereum/go-ethereum v1.9.5 => github.com/status-im/go-ethereum v1.9.5-status.4 @@ -29,9 +29,9 @@ require ( github.com/russolsen/transit v0.0.0-20180705123435-0794b4c4505a github.com/status-im/doubleratchet v2.0.0+incompatible github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 // indirect - github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7 + github.com/status-im/migrate/v4 v4.6.2-status.2 github.com/status-im/rendezvous v1.3.0 - github.com/status-im/status-protocol-go v0.3.1 + github.com/status-im/status-protocol-go v0.4.2 github.com/status-im/whisper v1.5.1 github.com/stretchr/testify v1.4.0 github.com/syndtr/goleveldb v1.0.0 diff --git a/go.sum b/go.sum index 4207764284..c36f6adb6b 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk= +github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -105,8 +107,8 @@ github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtf github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.0.0-20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 h1:mzrx39dGtGq0VEnTHjnakmczd4uFbhx2cZU3BJDsLdc= +github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -151,10 +153,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang-migrate/migrate v3.5.4+incompatible h1:R7OzwvCJTCgwapPCiX6DyBiu2czIUMDCB118gFTKTUA= -github.com/golang-migrate/migrate v3.5.4+incompatible/go.mod h1:IsVUlFN5puWOmXrqjgGUfIRIbU7mr8oNBE2tyERd9Wk= -github.com/golang-migrate/migrate/v4 v4.5.0 h1:ucd2qJu1BAKTtmjh7QlWYiq01DwlE/xXYQkOPE0ZTsI= -github.com/golang-migrate/migrate/v4 v4.5.0/go.mod h1:SzAcz2l+yDJVhQC7fwiF7T2MAFPMIkigJz98klRJ4OE= +github.com/golang-migrate/migrate/v4 v4.6.2 h1:LDDOHo/q1W5UDj6PbkxdCv7lv9yunyZHXvxuwDkGo3k= +github.com/golang-migrate/migrate/v4 v4.6.2/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -417,7 +417,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= @@ -555,13 +556,14 @@ github.com/status-im/go-multiaddr-ethv4 v1.2.0 h1:OT84UsUzTCwguqCpJqkrCMiL4VZ1Sv github.com/status-im/go-multiaddr-ethv4 v1.2.0/go.mod h1:2VQ3C+9zEurcceasz12gPAtmEzCeyLUGPeKLSXYQKHo= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed/go.mod h1:r8HggRBZ/k7TRwByq/Hp3P/ubFppIna0nvyavVK0pjA= -github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7 h1:gWtw0g41Y55lIOjUikkx0qjSfzl5lNZWhsvzWZIdtvQ= -github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7/go.mod h1:r8HggRBZ/k7TRwByq/Hp3P/ubFppIna0nvyavVK0pjA= +github.com/status-im/migrate/v4 v4.6.2-status.2 h1:SdC+sMDl/aI7vUlwD2qj2p7KsK4T60IS9z4/rYCCbI8= +github.com/status-im/migrate/v4 v4.6.2-status.2/go.mod h1:c/kc90n47GZu/58nnz1OMLTf7uE4Da4gZP5qmU+A/v8= github.com/status-im/rendezvous v1.3.0 h1:7RK/MXXW+tlm0asKm1u7Qp7Yni6AO29a7j8+E4Lbjg4= github.com/status-im/rendezvous v1.3.0/go.mod h1:+hzjuP+j/XzLPeF6E50b88pWOTLdTcwjvNYt+Gh1W1s= -github.com/status-im/status-protocol-go v0.3.1 h1:UrYdxKeW1GlFDtXWLyR66ebhUxmH1yWy46kal7gaSZ0= -github.com/status-im/status-protocol-go v0.3.1/go.mod h1:GyEipgAnT0gBMF57gN1yPAicFMM66VZEe2E8M5ktErI= +github.com/status-im/status-protocol-go v0.4.1-0.20191014103017-1f15a058b66c h1:e0WWnear7T8mXXmU7kKMOrnxGQ95AJOjDJTp58Do0Iw= +github.com/status-im/status-protocol-go v0.4.1-0.20191014103017-1f15a058b66c/go.mod h1:a1kbCiz4yIs2lxJEHsp5eFjdhB+5RUTN8+Uec6wWa4s= +github.com/status-im/status-protocol-go v0.4.2 h1:HAX51q3j+WAzbyeqryJEY1O6hqzTAbYEjkppIXkcNvg= +github.com/status-im/status-protocol-go v0.4.2/go.mod h1:WsfEnZo8dWadTbMqAoLztC/5p9SDwFeEGCmVDzSEmrQ= github.com/status-im/whisper v1.5.1 h1:87/XIg0Wjua7lXBGiEXgAfTOqlt2Q1dMDuxugTyZbbA= github.com/status-im/whisper v1.5.1/go.mod h1:emrOxzJme0k66QtbbQ2bdd3P8RCdLZ8sTD7SkwH1s2s= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= @@ -586,8 +588,10 @@ github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2 github.com/uber/jaeger-client-go v0.0.0-20180607151842-f7e0d4744fa6/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v0.0.0-20180615202729-a51202d6f4a7/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/vacp2p/mvds v0.0.21 h1:YeYja8noKsHvrnOHM4pqjBvDwwy+kUzXMkX1IBYJAbU= -github.com/vacp2p/mvds v0.0.21/go.mod h1:pIqr2Hg4cIkTJniGPCp4ptong2jxgxx6uToVoY94+II= +github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d h1:kc/9/ZbwyqIKQ2phzmPpBYGx7v2olwD2P7Uj8KKd+Bw= +github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d/go.mod h1:uUmtiahU7efOVl/5w5yk9jOze5xYpDZDrSrT8TvHXjQ= +github.com/vacp2p/mvds v0.0.23 h1:BKdn7tyGvl/J/Pwv6FlcW6Xbzm+17jv141GB1mFXyOU= +github.com/vacp2p/mvds v0.0.23/go.mod h1:uUmtiahU7efOVl/5w5yk9jOze5xYpDZDrSrT8TvHXjQ= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -607,6 +611,7 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= +go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= diff --git a/vendor/github.com/golang-migrate/migrate/v4/.travis.yml b/vendor/github.com/golang-migrate/migrate/v4/.travis.yml index 3a3ee81799..09183cc5b4 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/.travis.yml +++ b/vendor/github.com/golang-migrate/migrate/v4/.travis.yml @@ -26,8 +26,15 @@ cache: directories: - $GOPATH/pkg + before_install: - - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.16.0 + # Update docker to latest version: https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + # Install golangci-lint + - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.17.1 - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}" install: diff --git a/vendor/github.com/golang-migrate/migrate/v4/Dockerfile b/vendor/github.com/golang-migrate/migrate/v4/Dockerfile index d43c463d7a..cb11b0b2e7 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/Dockerfile +++ b/vendor/github.com/golang-migrate/migrate/v4/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.12-alpine3.9 AS downloader +FROM golang:1.12-alpine3.10 AS downloader ARG VERSION RUN apk add --no-cache git gcc musl-dev @@ -11,9 +11,9 @@ ENV GO111MODULE=on ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver" ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab" -RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate +RUN go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate -FROM alpine:3.9 +FROM alpine:3.10 RUN apk add --no-cache ca-certificates diff --git a/vendor/github.com/golang-migrate/migrate/v4/FAQ.md b/vendor/github.com/golang-migrate/migrate/v4/FAQ.md index 1142f59727..82ae4bf210 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/FAQ.md +++ b/vendor/github.com/golang-migrate/migrate/v4/FAQ.md @@ -68,3 +68,9 @@ Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses the `pg_advisory_lock` function. + +#### Do I need to create a table for tracking migration version used? +No, it is done automatically. + +#### Can I use migrate with a non-Go project? +Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework. \ No newline at end of file diff --git a/vendor/github.com/golang-migrate/migrate/v4/GETTING_STARTED.md b/vendor/github.com/golang-migrate/migrate/v4/GETTING_STARTED.md new file mode 100644 index 0000000000..41f293cc77 --- /dev/null +++ b/vendor/github.com/golang-migrate/migrate/v4/GETTING_STARTED.md @@ -0,0 +1,43 @@ +# Getting started +Before you start, you should understand the concept of forward/up and reverse/down database migrations. + +Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases) + +## Create migrations +Create some migrations using migrate CLI. Here is an example: +``` +migrate create -ext sql -dir db/migrations -seq create_users_table +``` +Once you create your files, you should fill them. + +**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first. +Developers and Teams should keep an eye on such cases (especially during code review). +[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more. + +Consider making your migrations idempotent - we can run the same sql code twice in a row with the same result. This makes our migrations more robust. On the other hand, it causes slightly less control over database schema - e.g. let's say you forgot to drop the table in down migration. You run down migration - the table is still there. When you run up migration again - `CREATE TABLE` would return an error, helping you find an issue in down migration, while `CREATE TABLE IF NOT EXISTS` would not. Use those conditions wisely. + +In case you would like to run several commands/queries in one migration, you should wrap them in a transaction (if your database supports it). +This way if one of commands fails, our database will remain unchanged. + +## Run migrations +Run your migrations through the CLI or your app and check if they applied expected changes. +Just to give you an idea: +``` +migrate -database YOUR_DATBASE_URL -path PATH_TO_YOUR_MIGRATIONS up +``` + +Just add the code to your app and you're ready to go! + +Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways. +(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again) +It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document. + +**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues. + +## Further reading: +- [PostgreSQL tutorial](database/postgres/TUTORIAL.md) +- [Best practices](MIGRATIONS.md) +- [FAQ](FAQ.md) +- Tools for testing your migrations in a container: + - https://github.com/dhui/dktest + - https://github.com/ory/dockertest diff --git a/vendor/github.com/golang-migrate/migrate/v4/README.md b/vendor/github.com/golang-migrate/migrate/v4/README.md index d251db0ab2..facd3c39fc 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/README.md +++ b/vendor/github.com/golang-migrate/migrate/v4/README.md @@ -5,6 +5,7 @@ [![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/) ![Supported Go Versions](https://img.shields.io/badge/Go-1.11%2C%201.12-lightgrey.svg) [![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases) +[![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate)](https://goreportcard.com/report/github.com/golang-migrate/migrate) # migrate @@ -139,6 +140,16 @@ func main() { } ``` +## Getting started + +Go to [getting started](GETTING_STARTED.md) + +## Tutorials + +- [PostgreSQL](database/postgres/TUTORIAL.md) + +(more tutorials to come) + ## Migration files Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-files-up-and-down-for-a-migration) diff --git a/vendor/github.com/golang-migrate/migrate/v4/database/driver.go b/vendor/github.com/golang-migrate/migrate/v4/database/driver.go index 901e5dd664..2c673caaff 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/database/driver.go +++ b/vendor/github.com/golang-migrate/migrate/v4/database/driver.go @@ -7,8 +7,9 @@ package database import ( "fmt" "io" - nurl "net/url" "sync" + + iurl "github.com/golang-migrate/migrate/v4/internal/url" ) var ( @@ -81,21 +82,16 @@ type Driver interface { // Open returns a new driver instance. func Open(url string) (Driver, error) { - u, err := nurl.Parse(url) + scheme, err := iurl.SchemeFromURL(url) if err != nil { - return nil, fmt.Errorf("Unable to parse URL. Did you escape all reserved URL characters? "+ - "See: https://github.com/golang-migrate/migrate#database-urls Error: %v", err) - } - - if u.Scheme == "" { - return nil, fmt.Errorf("database driver: invalid URL scheme") + return nil, err } driversMu.RLock() - d, ok := drivers[u.Scheme] + d, ok := drivers[scheme] driversMu.RUnlock() if !ok { - return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", u.Scheme) + return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", scheme) } return d.Open(url) diff --git a/vendor/github.com/golang-migrate/migrate/v4/go.mod b/vendor/github.com/golang-migrate/migrate/v4/go.mod index ae6e81879d..7729edcc2b 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/go.mod +++ b/vendor/github.com/golang-migrate/migrate/v4/go.mod @@ -7,10 +7,11 @@ require ( github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cockroachdb/apd v1.1.0 // indirect github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c + github.com/containerd/containerd v1.2.7 // indirect github.com/cznic/ql v1.2.0 github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 github.com/dhui/dktest v0.3.0 - github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf + github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 github.com/fsouza/fake-gcs-server v1.7.0 github.com/go-sql-driver/mysql v1.4.1 github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 @@ -28,7 +29,7 @@ require ( github.com/kshvakov/clickhouse v1.3.5 github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 - github.com/mongodb/mongo-go-driver v0.3.0 + github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 github.com/pkg/errors v0.8.1 // indirect github.com/satori/go.uuid v1.2.0 // indirect @@ -40,6 +41,7 @@ require ( github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect github.com/xdg/stringprep v1.0.0 // indirect gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect + go.mongodb.org/mongo-driver v1.1.0 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect diff --git a/vendor/github.com/golang-migrate/migrate/v4/go.sum b/vendor/github.com/golang-migrate/migrate/v4/go.sum index 55002b3215..d7944eafe3 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/go.sum +++ b/vendor/github.com/golang-migrate/migrate/v4/go.sum @@ -26,6 +26,8 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c h1:2zRrJWIt/f9c9HhNHAgrRgq0San5gRRUJTBXLkchal0= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk= +github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas= @@ -56,8 +58,8 @@ github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 h1:mzrx39dGtGq0VEnTHjnakmczd4uFbhx2cZU3BJDsLdc= +github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -147,8 +149,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mongodb/mongo-go-driver v0.3.0 h1:00tKWMrabkVU1e57/TTP4ZBIfhn/wmjlSiRnIM9d0T8= -github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= @@ -196,6 +198,8 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= +go.mongodb.org/mongo-driver v1.1.0 h1:aeOqSrhl9eDRAap/3T5pCfMBEBxZ0vuXBP+RMtp2KX8= +go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= diff --git a/vendor/github.com/golang-migrate/migrate/v4/internal/url/url.go b/vendor/github.com/golang-migrate/migrate/v4/internal/url/url.go new file mode 100644 index 0000000000..e793fa828e --- /dev/null +++ b/vendor/github.com/golang-migrate/migrate/v4/internal/url/url.go @@ -0,0 +1,25 @@ +package url + +import ( + "errors" + "strings" +) + +var errNoScheme = errors.New("no scheme") +var errEmptyURL = errors.New("URL cannot be empty") + +// schemeFromURL returns the scheme from a URL string +func SchemeFromURL(url string) (string, error) { + if url == "" { + return "", errEmptyURL + } + + i := strings.Index(url, ":") + + // No : or : is the first character. + if i < 1 { + return "", errNoScheme + } + + return url[0:i], nil +} diff --git a/vendor/github.com/golang-migrate/migrate/v4/migrate.go b/vendor/github.com/golang-migrate/migrate/v4/migrate.go index 3ede504e52..1478bc2a00 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/migrate.go +++ b/vendor/github.com/golang-migrate/migrate/v4/migrate.go @@ -7,12 +7,14 @@ package migrate import ( "errors" "fmt" - "github.com/hashicorp/go-multierror" "os" "sync" "time" + "github.com/hashicorp/go-multierror" + "github.com/golang-migrate/migrate/v4/database" + iurl "github.com/golang-migrate/migrate/v4/internal/url" "github.com/golang-migrate/migrate/v4/source" ) @@ -85,13 +87,13 @@ type Migrate struct { func New(sourceURL, databaseURL string) (*Migrate, error) { m := newCommon() - sourceName, err := sourceSchemeFromURL(sourceURL) + sourceName, err := iurl.SchemeFromURL(sourceURL) if err != nil { return nil, err } m.sourceName = sourceName - databaseName, err := databaseSchemeFromURL(databaseURL) + databaseName, err := iurl.SchemeFromURL(databaseURL) if err != nil { return nil, err } @@ -119,7 +121,7 @@ func New(sourceURL, databaseURL string) (*Migrate, error) { func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInstance database.Driver) (*Migrate, error) { m := newCommon() - sourceName, err := schemeFromURL(sourceURL) + sourceName, err := iurl.SchemeFromURL(sourceURL) if err != nil { return nil, err } @@ -145,7 +147,7 @@ func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInst func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, databaseURL string) (*Migrate, error) { m := newCommon() - databaseName, err := schemeFromURL(databaseURL) + databaseName, err := iurl.SchemeFromURL(databaseURL) if err != nil { return nil, err } @@ -804,6 +806,7 @@ func (m *Migrate) versionExists(version uint) (result error) { return err } + m.logErr(fmt.Errorf("no migration found for version %d", version)) return os.ErrNotExist } diff --git a/vendor/github.com/golang-migrate/migrate/v4/util.go b/vendor/github.com/golang-migrate/migrate/v4/util.go index 1cef03cd81..26131a3ffa 100644 --- a/vendor/github.com/golang-migrate/migrate/v4/util.go +++ b/vendor/github.com/golang-migrate/migrate/v4/util.go @@ -1,7 +1,6 @@ package migrate import ( - "errors" "fmt" nurl "net/url" "strings" @@ -49,42 +48,6 @@ func suint(n int) uint { return uint(n) } -var errNoScheme = errors.New("no scheme") -var errEmptyURL = errors.New("URL cannot be empty") - -func sourceSchemeFromURL(url string) (string, error) { - u, err := schemeFromURL(url) - if err != nil { - return "", fmt.Errorf("source: %v", err) - } - return u, nil -} - -func databaseSchemeFromURL(url string) (string, error) { - u, err := schemeFromURL(url) - if err != nil { - return "", fmt.Errorf("database: %v", err) - } - return u, nil -} - -// schemeFromURL returns the scheme from a URL string -func schemeFromURL(url string) (string, error) { - if url == "" { - return "", errEmptyURL - } - - u, err := nurl.Parse(url) - if err != nil { - return "", err - } - if len(u.Scheme) == 0 { - return "", errNoScheme - } - - return u.Scheme, nil -} - // FilterCustomQuery filters all query values starting with `x-` func FilterCustomQuery(u *nurl.URL) *nurl.URL { ux := *u diff --git a/vendor/github.com/status-im/migrate/v4/.golangci.yml b/vendor/github.com/status-im/migrate/v4/.golangci.yml index 2d4d89158b..f447ee1ce0 100644 --- a/vendor/github.com/status-im/migrate/v4/.golangci.yml +++ b/vendor/github.com/status-im/migrate/v4/.golangci.yml @@ -1,6 +1,6 @@ run: # timeout for analysis, e.g. 30s, 5m, default is 1m - deadline: 2m + timeout: 2m linters: enable: #- golint diff --git a/vendor/github.com/status-im/migrate/v4/.travis.yml b/vendor/github.com/status-im/migrate/v4/.travis.yml index 3a3ee81799..cf8c68a041 100644 --- a/vendor/github.com/status-im/migrate/v4/.travis.yml +++ b/vendor/github.com/status-im/migrate/v4/.travis.yml @@ -6,8 +6,8 @@ matrix: - go: master include: # Supported versions of Go: https://golang.org/dl/ - - go: "1.11.x" - go: "1.12.x" + - go: "1.13.x" - go: master go_import_path: github.com/golang-migrate/migrate @@ -26,8 +26,15 @@ cache: directories: - $GOPATH/pkg + before_install: - - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.16.0 + # Update docker to latest version: https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + # Install golangci-lint + - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.20.0 - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}" install: diff --git a/vendor/github.com/status-im/migrate/v4/Dockerfile b/vendor/github.com/status-im/migrate/v4/Dockerfile index f97ccc06d3..cb11b0b2e7 100644 --- a/vendor/github.com/status-im/migrate/v4/Dockerfile +++ b/vendor/github.com/status-im/migrate/v4/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.12-alpine3.9 AS downloader +FROM golang:1.12-alpine3.10 AS downloader ARG VERSION RUN apk add --no-cache git gcc musl-dev @@ -9,11 +9,11 @@ COPY . ./ ENV GO111MODULE=on ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver" -ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab" +ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab" -RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate +RUN go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate -FROM alpine:3.9 +FROM alpine:3.10 RUN apk add --no-cache ca-certificates diff --git a/vendor/github.com/status-im/migrate/v4/FAQ.md b/vendor/github.com/status-im/migrate/v4/FAQ.md index 1142f59727..82ae4bf210 100644 --- a/vendor/github.com/status-im/migrate/v4/FAQ.md +++ b/vendor/github.com/status-im/migrate/v4/FAQ.md @@ -68,3 +68,9 @@ Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses the `pg_advisory_lock` function. + +#### Do I need to create a table for tracking migration version used? +No, it is done automatically. + +#### Can I use migrate with a non-Go project? +Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework. \ No newline at end of file diff --git a/vendor/github.com/status-im/migrate/v4/GETTING_STARTED.md b/vendor/github.com/status-im/migrate/v4/GETTING_STARTED.md new file mode 100644 index 0000000000..41f293cc77 --- /dev/null +++ b/vendor/github.com/status-im/migrate/v4/GETTING_STARTED.md @@ -0,0 +1,43 @@ +# Getting started +Before you start, you should understand the concept of forward/up and reverse/down database migrations. + +Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases) + +## Create migrations +Create some migrations using migrate CLI. Here is an example: +``` +migrate create -ext sql -dir db/migrations -seq create_users_table +``` +Once you create your files, you should fill them. + +**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first. +Developers and Teams should keep an eye on such cases (especially during code review). +[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more. + +Consider making your migrations idempotent - we can run the same sql code twice in a row with the same result. This makes our migrations more robust. On the other hand, it causes slightly less control over database schema - e.g. let's say you forgot to drop the table in down migration. You run down migration - the table is still there. When you run up migration again - `CREATE TABLE` would return an error, helping you find an issue in down migration, while `CREATE TABLE IF NOT EXISTS` would not. Use those conditions wisely. + +In case you would like to run several commands/queries in one migration, you should wrap them in a transaction (if your database supports it). +This way if one of commands fails, our database will remain unchanged. + +## Run migrations +Run your migrations through the CLI or your app and check if they applied expected changes. +Just to give you an idea: +``` +migrate -database YOUR_DATBASE_URL -path PATH_TO_YOUR_MIGRATIONS up +``` + +Just add the code to your app and you're ready to go! + +Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways. +(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again) +It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document. + +**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues. + +## Further reading: +- [PostgreSQL tutorial](database/postgres/TUTORIAL.md) +- [Best practices](MIGRATIONS.md) +- [FAQ](FAQ.md) +- Tools for testing your migrations in a container: + - https://github.com/dhui/dktest + - https://github.com/ory/dockertest diff --git a/vendor/github.com/status-im/migrate/v4/MIGRATIONS.md b/vendor/github.com/status-im/migrate/v4/MIGRATIONS.md index 8363887029..3475d8e4ea 100644 --- a/vendor/github.com/status-im/migrate/v4/MIGRATIONS.md +++ b/vendor/github.com/status-im/migrate/v4/MIGRATIONS.md @@ -44,9 +44,14 @@ It is suggested that the version number of corresponding `up` and `down` migrati files be equivalent for clarity, but they are allowed to differ so long as the relative ordering of the migrations is preserved. -The migration files are permitted to be empty, so in the event that a migration -is a no-op or is irreversible, it is recommended to still include both migration -files, and either leaving them empty or adding a comment as appropriate. +The migration files are permitted to be "empty", in the event that a migration +is a no-op or is irreversible. It is recommended to still include both migration +files by making the whole migration file consist of a comment. +If your database does not support comments, then deleting the migration file will also work. +Note, an actual empty file (e.g. a 0 byte file) may cause issues with your database since migrate +will attempt to run an empty query. In this case, deleting the migration file will also work. +For the rational of this behavior see: +[#244 (comment)](https://github.com/golang-migrate/migrate/issues/244#issuecomment-510758270) ## Migration Content Format diff --git a/vendor/github.com/status-im/migrate/v4/Makefile b/vendor/github.com/status-im/migrate/v4/Makefile index 3bc0de95af..a0cd24afeb 100644 --- a/vendor/github.com/status-im/migrate/v4/Makefile +++ b/vendor/github.com/status-im/migrate/v4/Makefile @@ -1,4 +1,4 @@ -SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab +SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= @@ -27,27 +27,14 @@ test-short: test: @-rm -r $(COVERAGE_DIR) @mkdir $(COVERAGE_DIR) - make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/_$$(RAND).txt -bench=. -benchmem -timeout 20m' - @echo 'mode: atomic' > $(COVERAGE_DIR)/combined.txt - @cat $(COVERAGE_DIR)/_*.txt | grep -v 'mode: atomic' >> $(COVERAGE_DIR)/combined.txt + make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/combined.txt -bench=. -benchmem -timeout 20m' test-with-flags: - @echo SOURCE: $(SOURCE) + @echo SOURCE: $(SOURCE) @echo DATABASE: $(DATABASE) - @go test $(TEST_FLAGS) . - @go test $(TEST_FLAGS) ./cli/... - @go test $(TEST_FLAGS) ./database - @go test $(TEST_FLAGS) ./testing/... - - @echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{} - @go test $(TEST_FLAGS) ./source/testing/... - @go test $(TEST_FLAGS) ./source/stub/... - - @echo -n '$(DATABASE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./database/{} - @go test $(TEST_FLAGS) ./database/testing/... - @go test $(TEST_FLAGS) ./database/stub/... + @go test $(TEST_FLAGS) ./... kill-orphaned-docker-containers: @@ -84,7 +71,7 @@ rewrite-import-paths: docs: -make kill-docs nohup godoc -play -http=127.0.0.1:6064 /dev/null 2>&1 & echo $$! > .godoc.pid - cat .godoc.pid + cat .godoc.pid kill-docs: diff --git a/vendor/github.com/status-im/migrate/v4/README.md b/vendor/github.com/status-im/migrate/v4/README.md index 0b38a5e38b..6be55a90ac 100644 --- a/vendor/github.com/status-im/migrate/v4/README.md +++ b/vendor/github.com/status-im/migrate/v4/README.md @@ -3,19 +3,19 @@ [![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master) [![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs) [![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/) -![Supported Go Versions](https://img.shields.io/badge/Go-1.11%2C%201.12-lightgrey.svg) +![Supported Go Versions](https://img.shields.io/badge/Go-1.12%2C%201.13-lightgrey.svg) [![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases) - +[![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate)](https://goreportcard.com/report/github.com/golang-migrate/migrate) # migrate __Database migrations written in Go. Use as [CLI](#cli-usage) or import as [library](#use-in-your-go-project).__ - * Migrate reads migrations from [sources](#migration-sources) +* Migrate reads migrations from [sources](#migration-sources) and applies them in correct order to a [database](#databases). - * Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof. +* Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof. (Keeps the drivers lightweight, too.) - * Database drivers don't assume things or try to correct user input. When in doubt, fail. +* Database drivers don't assume things or try to correct user input. When in doubt, fail. Forked from [mattes/migrate](https://github.com/mattes/migrate) @@ -23,21 +23,21 @@ Forked from [mattes/migrate](https://github.com/mattes/migrate) Database drivers run migrations. [Add a new database?](database/driver.go) - * [PostgreSQL](database/postgres) - * [Redshift](database/redshift) - * [Ql](database/ql) - * [Cassandra](database/cassandra) - * [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165)) - * [MySQL/ MariaDB](database/mysql) - * [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167)) - * [MongoDB](database/mongodb) - * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) - * [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) - * [Google Cloud Spanner](database/spanner) - * [CockroachDB](database/cockroachdb) - * [ClickHouse](database/clickhouse) - * [Firebird](database/firebird) ([todo #49](https://github.com/golang-migrate/migrate/issues/49)) - * [MS SQL Server](database/sqlserver) +* [PostgreSQL](database/postgres) +* [Redshift](database/redshift) +* [Ql](database/ql) +* [Cassandra](database/cassandra) +* [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165)) +* [MySQL/ MariaDB](database/mysql) +* [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167)) +* [MongoDB](database/mongodb) +* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) +* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) +* [Google Cloud Spanner](database/spanner) +* [CockroachDB](database/cockroachdb) +* [ClickHouse](database/clickhouse) +* [Firebird](database/firebird) ([todo #49](https://github.com/golang-migrate/migrate/issues/49)) +* [MS SQL Server](database/sqlserver) ### Database URLs @@ -49,6 +49,7 @@ Explicitly, the following characters need to be escaped: `!`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `/`, `:`, `;`, `=`, `?`, `@`, `[`, `]` It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python snippets below: + ```bash $ python3 -c 'import urllib.parse; print(urllib.parse.quote(input("String to encode: "), ""))' String to encode: FAKEpassword!#$%&'()*+,/:;=?@[] @@ -63,44 +64,43 @@ $ Source drivers read migrations from local or remote sources. [Add a new source?](source/driver.go) - * [Filesystem](source/file) - read from fileystem - * [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata)) - * [Github](source/github) - read from remote Github repositories - * [Gitlab](source/gitlab) - read from remote Gitlab repositories - * [AWS S3](source/aws_s3) - read from Amazon Web Services S3 - * [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage - - +* [Filesystem](source/file) - read from filesystem +* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata)) +* [Github](source/github) - read from remote Github repositories +* [Github Enterprise](source/github_ee) - read from remote Github Enterprise repositories +* [Gitlab](source/gitlab) - read from remote Gitlab repositories +* [AWS S3](source/aws_s3) - read from Amazon Web Services S3 +* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage ## CLI usage - * Simple wrapper around this library. - * Handles ctrl+c (SIGINT) gracefully. - * No config search paths, no config files, no magic ENV var injections. +* Simple wrapper around this library. +* Handles ctrl+c (SIGINT) gracefully. +* No config search paths, no config files, no magic ENV var injections. -__[CLI Documentation](cli)__ +__[CLI Documentation](cmd/migrate)__ -### Basic usage: +### Basic usage -``` +```bash $ migrate -source file://path/to/migrations -database postgres://localhost:5432/database up 2 ``` ### Docker usage -``` +```bash $ docker run -v {{ migration dir }}:/migrations --network host migrate/migrate -path=/migrations/ -database postgres://localhost:5432/database up 2 ``` ## Use in your Go project - * API is stable and frozen for this release (v3 & v4). - * Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies. - * To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`. - * Bring your own logger. - * Uses `io.Reader` streams internally for low memory overhead. - * Thread-safe and no goroutine leaks. +* API is stable and frozen for this release (v3 & v4). +* Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies. +* To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`. +* Bring your own logger. +* Uses `io.Reader` streams internally for low memory overhead. +* Thread-safe and no goroutine leaks. __[Go Documentation](https://godoc.org/github.com/golang-migrate/migrate)__ @@ -140,11 +140,21 @@ func main() { } ``` +## Getting started + +Go to [getting started](GETTING_STARTED.md) + +## Tutorials + +- [PostgreSQL](database/postgres/TUTORIAL.md) + +(more tutorials to come) + ## Migration files Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-files-up-and-down-for-a-migration) -``` +```bash 1481574547_create_users_table.up.sql 1481574547_create_users_table.down.sql ``` @@ -166,8 +176,6 @@ read the [development guide](CONTRIBUTING.md). Also have a look at the [FAQ](FAQ.md). - - --- Looking for alternatives? [https://awesome-go.com/#database](https://awesome-go.com/#database). diff --git a/vendor/github.com/status-im/migrate/v4/database/postgres/TUTORIAL.md b/vendor/github.com/status-im/migrate/v4/database/postgres/TUTORIAL.md new file mode 100644 index 0000000000..8b06a48642 --- /dev/null +++ b/vendor/github.com/status-im/migrate/v4/database/postgres/TUTORIAL.md @@ -0,0 +1,148 @@ +# PostgreSQL tutorial for beginners + +## Create/configure database + +For the purpose of this tutorial let's create PostgreSQL database called `example`. +Our user here is `postgres`, password `password`, and host is `localhost`. +``` +psql -h localhost -U postgres -w -c "create database example;" +``` +When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convienience: +``` +export POSTGRESQL_URL=postgres://postgres:password@localhost:5432/example?sslmode=disable +``` +`sslmode=disable` means that the connection with our database will not be encrypted. Enabling it is left as an exercise. + +You can find further description of database URLs [here](README.md#database-urls). + +## Create migrations +Let's create table called `users`: +``` +migrate create -ext sql -dir db/migrations -seq create_users_table +``` +If there were no errors, we should have two files available under `db/migrations` folder: +- 000001_create_users_table.down.sql +- 000001_create_users_table.up.sql + +Note the `sql` extension that we provided. + +In the `.up.sql` file let's create the table: +``` +CREATE TABLE IF NOT EXISTS users( + user_id serial PRIMARY KEY, + username VARCHAR (50) UNIQUE NOT NULL, + password VARCHAR (50) NOT NULL, + email VARCHAR (300) UNIQUE NOT NULL +); +``` +And in the `.down.sql` let's delete it: +``` +DROP TABLE IF EXISTS users; +``` +By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](GETTING_STARTED.md#create-migrations) + +## Run migrations +``` +migrate -database ${POSTGRESQL_URL} -path db/migrations up +``` +Let's check if the table was created properly by running `psql example -c "\d users"`. +The output you are supposed to see: +``` + Table "public.users" + Column | Type | Modifiers +----------+------------------------+--------------------------------------------------------- + user_id | integer | not null default nextval('users_user_id_seq'::regclass) + username | character varying(50) | not null + password | character varying(50) | not null + email | character varying(300) | not null +Indexes: + "users_pkey" PRIMARY KEY, btree (user_id) + "users_email_key" UNIQUE CONSTRAINT, btree (email) + "users_username_key" UNIQUE CONSTRAINT, btree (username) +``` +Great! Now let's check if running reverse migration also works: +``` +migrate -database ${POSTGRESQL_URL} -path db/migrations down +``` +Make sure to check if your database changed as expected in this case as well. + +## Database transactions + +To show database transactions usage, let's create another set of migrations by running: +``` +migrate create -ext sql -dir db/migrations -seq add_mood_to_users +``` +Again, it should create for us two migrations files: +- 000002_add_mood_to_users.down.sql +- 000002_add_mood_to_users.up.sql + +In Postgres, when we want our queries to be done in a transaction, we need to wrap it with `BEGIN` and `COMMIT` commands. +In our example, we are going to add a column to our database that can only accept enumerable values or NULL. +Migration up: +``` +BEGIN; + +CREATE TYPE enum_mood AS ENUM ( + 'happy', + 'sad', + 'neutral' +); +ALTER TABLE users ADD COLUMN mood enum_mood; + +COMMIT; +``` +Migration down: +``` +BEGIN; + +ALTER TABLE users DROP COLUMN mood; +DROP TYPE enum_mood; + +COMMIT; +``` + +Now we can run our new migration and check the database: +``` +migrate -database ${POSTGRESQL_URL} -path db/migrations up +psql example -c "\d users" +``` +Expected output: +``` + Table "public.users" + Column | Type | Modifiers +----------+------------------------+--------------------------------------------------------- + user_id | integer | not null default nextval('users_user_id_seq'::regclass) + username | character varying(50) | not null + password | character varying(50) | not null + email | character varying(300) | not null + mood | enum_mood | +Indexes: + "users_pkey" PRIMARY KEY, btree (user_id) + "users_email_key" UNIQUE CONSTRAINT, btree (email) + "users_username_key" UNIQUE CONSTRAINT, btree (username) +``` + +## Optional: Run migrations within your Go app +Here is a very simple app running migrations for the above configuration: +``` +import ( + "log" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +func main() { + m, err := migrate.New( + "file://db/migrations", + "postgres://postgres:postgres@localhost:5432/example?sslmode=disable") + if err != nil { + log.Fatal(err) + } + if err := m.Up(); err != nil { + log.Fatal(err) + } +} +``` +You can find details [here](README.md#use-in-your-go-project) \ No newline at end of file diff --git a/vendor/github.com/status-im/migrate/v4/database/postgres/postgres.go b/vendor/github.com/status-im/migrate/v4/database/postgres/postgres.go index 4c0cc1c402..7c334921aa 100644 --- a/vendor/github.com/status-im/migrate/v4/database/postgres/postgres.go +++ b/vendor/github.com/status-im/migrate/v4/database/postgres/postgres.go @@ -150,8 +150,7 @@ func (p *Postgres) Lock() error { return err } - // This will either obtain the lock immediately and return true, - // or return false if the lock cannot be acquired immediately. + // This will wait indefinitely until the lock can be acquired. query := `SELECT pg_advisory_lock($1)` if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil { return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} diff --git a/vendor/github.com/status-im/migrate/v4/go.mod b/vendor/github.com/status-im/migrate/v4/go.mod index 4740e3f8da..951a51b68e 100644 --- a/vendor/github.com/status-im/migrate/v4/go.mod +++ b/vendor/github.com/status-im/migrate/v4/go.mod @@ -7,10 +7,11 @@ require ( github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cockroachdb/apd v1.1.0 // indirect github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c + github.com/containerd/containerd v1.2.7 // indirect github.com/cznic/ql v1.2.0 github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 github.com/dhui/dktest v0.3.0 - github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf + github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 github.com/fsouza/fake-gcs-server v1.7.0 github.com/go-sql-driver/mysql v1.4.1 github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 @@ -28,7 +29,7 @@ require ( github.com/kshvakov/clickhouse v1.3.5 github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 - github.com/mongodb/mongo-go-driver v0.3.0 + github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 github.com/pkg/errors v0.8.1 // indirect github.com/satori/go.uuid v1.2.0 // indirect @@ -40,6 +41,7 @@ require ( github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect github.com/xdg/stringprep v1.0.0 // indirect gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect + go.mongodb.org/mongo-driver v1.1.0 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect @@ -52,3 +54,5 @@ require ( google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb google.golang.org/grpc v1.20.1 // indirect ) + +go 1.12 diff --git a/vendor/github.com/status-im/migrate/v4/go.sum b/vendor/github.com/status-im/migrate/v4/go.sum index 55002b3215..d7944eafe3 100644 --- a/vendor/github.com/status-im/migrate/v4/go.sum +++ b/vendor/github.com/status-im/migrate/v4/go.sum @@ -26,6 +26,8 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c h1:2zRrJWIt/f9c9HhNHAgrRgq0San5gRRUJTBXLkchal0= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk= +github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas= @@ -56,8 +58,8 @@ github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 h1:mzrx39dGtGq0VEnTHjnakmczd4uFbhx2cZU3BJDsLdc= +github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -147,8 +149,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mongodb/mongo-go-driver v0.3.0 h1:00tKWMrabkVU1e57/TTP4ZBIfhn/wmjlSiRnIM9d0T8= -github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= @@ -196,6 +198,8 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= +go.mongodb.org/mongo-driver v1.1.0 h1:aeOqSrhl9eDRAap/3T5pCfMBEBxZ0vuXBP+RMtp2KX8= +go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= diff --git a/vendor/github.com/status-im/migrate/v4/internal/url/url.go b/vendor/github.com/status-im/migrate/v4/internal/url/url.go new file mode 100644 index 0000000000..e793fa828e --- /dev/null +++ b/vendor/github.com/status-im/migrate/v4/internal/url/url.go @@ -0,0 +1,25 @@ +package url + +import ( + "errors" + "strings" +) + +var errNoScheme = errors.New("no scheme") +var errEmptyURL = errors.New("URL cannot be empty") + +// schemeFromURL returns the scheme from a URL string +func SchemeFromURL(url string) (string, error) { + if url == "" { + return "", errEmptyURL + } + + i := strings.Index(url, ":") + + // No : or : is the first character. + if i < 1 { + return "", errNoScheme + } + + return url[0:i], nil +} diff --git a/vendor/github.com/status-im/migrate/v4/migrate.go b/vendor/github.com/status-im/migrate/v4/migrate.go index a4a694b400..fd3c325568 100644 --- a/vendor/github.com/status-im/migrate/v4/migrate.go +++ b/vendor/github.com/status-im/migrate/v4/migrate.go @@ -7,12 +7,14 @@ package migrate import ( "errors" "fmt" - "github.com/hashicorp/go-multierror" "os" "sync" "time" + "github.com/hashicorp/go-multierror" + "github.com/golang-migrate/migrate/v4/database" + iurl "github.com/status-im/migrate/v4/internal/url" "github.com/golang-migrate/migrate/v4/source" ) @@ -85,13 +87,13 @@ type Migrate struct { func New(sourceURL, databaseURL string) (*Migrate, error) { m := newCommon() - sourceName, err := sourceSchemeFromURL(sourceURL) + sourceName, err := iurl.SchemeFromURL(sourceURL) if err != nil { return nil, err } m.sourceName = sourceName - databaseName, err := databaseSchemeFromURL(databaseURL) + databaseName, err := iurl.SchemeFromURL(databaseURL) if err != nil { return nil, err } @@ -119,7 +121,7 @@ func New(sourceURL, databaseURL string) (*Migrate, error) { func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInstance database.Driver) (*Migrate, error) { m := newCommon() - sourceName, err := schemeFromURL(sourceURL) + sourceName, err := iurl.SchemeFromURL(sourceURL) if err != nil { return nil, err } @@ -145,7 +147,7 @@ func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInst func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, databaseURL string) (*Migrate, error) { m := newCommon() - databaseName, err := schemeFromURL(databaseURL) + databaseName, err := iurl.SchemeFromURL(databaseURL) if err != nil { return nil, err } @@ -804,6 +806,7 @@ func (m *Migrate) versionExists(version uint) (result error) { return err } + m.logErr(fmt.Errorf("no migration found for version %d", version)) return os.ErrNotExist } @@ -950,7 +953,7 @@ func (m *Migrate) unlock() error { // if a prevErr is not nil. func (m *Migrate) unlockErr(prevErr error) error { if err := m.unlock(); err != nil { - return NewMultiError(prevErr, err) + return multierror.Append(prevErr, err) } return prevErr } diff --git a/vendor/github.com/status-im/migrate/v4/util.go b/vendor/github.com/status-im/migrate/v4/util.go index 4aeb0f2211..26131a3ffa 100644 --- a/vendor/github.com/status-im/migrate/v4/util.go +++ b/vendor/github.com/status-im/migrate/v4/util.go @@ -1,18 +1,22 @@ package migrate import ( - "errors" "fmt" nurl "net/url" "strings" ) // MultiError holds multiple errors. +// +// Deprecated: Use github.com/hashicorp/go-multierror instead type MultiError struct { Errs []error } // NewMultiError returns an error type holding multiple errors. +// +// Deprecated: Use github.com/hashicorp/go-multierror instead +// func NewMultiError(errs ...error) MultiError { compactErrs := make([]error, 0) for _, e := range errs { @@ -44,42 +48,6 @@ func suint(n int) uint { return uint(n) } -var errNoScheme = errors.New("no scheme") -var errEmptyURL = errors.New("URL cannot be empty") - -func sourceSchemeFromURL(url string) (string, error) { - u, err := schemeFromURL(url) - if err != nil { - return "", fmt.Errorf("source: %v", err) - } - return u, nil -} - -func databaseSchemeFromURL(url string) (string, error) { - u, err := schemeFromURL(url) - if err != nil { - return "", fmt.Errorf("database: %v", err) - } - return u, nil -} - -// schemeFromURL returns the scheme from a URL string -func schemeFromURL(url string) (string, error) { - if url == "" { - return "", errEmptyURL - } - - u, err := nurl.Parse(url) - if err != nil { - return "", err - } - if len(u.Scheme) == 0 { - return "", errNoScheme - } - - return u.Scheme, nil -} - // FilterCustomQuery filters all query values starting with `x-` func FilterCustomQuery(u *nurl.URL) *nurl.URL { ux := *u diff --git a/vendor/github.com/status-im/status-protocol-go/chat.go b/vendor/github.com/status-im/status-protocol-go/chat.go index 70531a1f82..a892163882 100644 --- a/vendor/github.com/status-im/status-protocol-go/chat.go +++ b/vendor/github.com/status-im/status-protocol-go/chat.go @@ -2,10 +2,13 @@ package statusproto import ( "crypto/ecdsa" + "crypto/sha1" + "encoding/hex" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" - "github.com/google/uuid" statusproto "github.com/status-im/status-protocol-go/types" + protocol "github.com/status-im/status-protocol-go/v1" ) type ChatType int @@ -52,6 +55,55 @@ type Chat struct { MembershipUpdates []ChatMembershipUpdate `json:"membershipUpdates"` } +func (c *Chat) MembersAsPublicKeys() ([]*ecdsa.PublicKey, error) { + publicKeys := make([]string, len(c.Members)) + for idx, item := range c.Members { + publicKeys[idx] = item.ID + } + return stringSliceToPublicKeys(publicKeys, true) +} + +func (c *Chat) updateChatFromProtocolGroup(g *protocol.Group) { + // ID + c.ID = g.ChatID() + + // Name + c.Name = g.Name() + + // Members + members := g.Members() + admins := g.Admins() + joined := g.Joined() + chatMembers := make([]ChatMember, 0, len(members)) + for _, m := range members { + chatMember := ChatMember{ + ID: m, + } + chatMember.Admin = stringSliceContains(admins, m) + chatMember.Joined = stringSliceContains(joined, m) + chatMembers = append(chatMembers, chatMember) + } + c.Members = chatMembers + + // MembershipUpdates + updates := g.Updates() + membershipUpdates := make([]ChatMembershipUpdate, 0, len(updates)) + for _, update := range updates { + membershipUpdate := ChatMembershipUpdate{ + Type: update.Type, + Name: update.Name, + ClockValue: uint64(update.ClockValue), // TODO: get rid of type casting + Signature: update.Signature, + From: update.From, + Member: update.Member, + Members: update.Members, + } + membershipUpdate.setID() + membershipUpdates = append(membershipUpdates, membershipUpdate) + } + c.MembershipUpdates = membershipUpdates +} + // ChatMembershipUpdate represent an event on membership of the chat type ChatMembershipUpdate struct { // Unique identifier for the event @@ -72,6 +124,11 @@ type ChatMembershipUpdate struct { Members []string `json:"members,omitempty"` } +func (u *ChatMembershipUpdate) setID() { + sum := sha1.Sum([]byte(u.Signature)) + u.ID = hex.EncodeToString(sum[:]) +} + // ChatMember represents a member who participates in a group chat type ChatMember struct { // ID is the hex encoded public key of the member @@ -113,14 +170,8 @@ func CreatePublicChat(name string) Chat { } } -func groupChatID(creator *ecdsa.PublicKey) string { - return uuid.New().String() + statusproto.EncodeHex(crypto.FromECDSAPub(creator)) -} - -func CreateGroupChat(name string, creator *ecdsa.PublicKey) Chat { +func createGroupChat() Chat { return Chat{ - ID: groupChatID(creator), - Name: name, Active: true, ChatType: ChatTypePrivateGroupChat, } @@ -134,3 +185,35 @@ func findChatByID(chatID string, chats []*Chat) *Chat { } return nil } + +func stringSliceToPublicKeys(slice []string, prefixed bool) ([]*ecdsa.PublicKey, error) { + result := make([]*ecdsa.PublicKey, len(slice)) + for idx, item := range slice { + var ( + b []byte + err error + ) + if prefixed { + b, err = hexutil.Decode(item) + } else { + b, err = hex.DecodeString(item) + } + if err != nil { + return nil, err + } + result[idx], err = crypto.UnmarshalPubkey(b) + if err != nil { + return nil, err + } + } + return result, nil +} + +func stringSliceContains(slice []string, item string) bool { + for _, s := range slice { + if s == item { + return true + } + } + return false +} diff --git a/vendor/github.com/status-im/status-protocol-go/chat_group_proxy.go b/vendor/github.com/status-im/status-protocol-go/chat_group_proxy.go new file mode 100644 index 0000000000..19e4aee01a --- /dev/null +++ b/vendor/github.com/status-im/status-protocol-go/chat_group_proxy.go @@ -0,0 +1,28 @@ +package statusproto + +import ( + protocol "github.com/status-im/status-protocol-go/v1" +) + +func newProtocolGroupFromChat(chat *Chat) (*protocol.Group, error) { + return protocol.NewGroup(chat.ID, chatToFlattenMembershipUpdate(chat)) +} + +func chatToFlattenMembershipUpdate(chat *Chat) []protocol.MembershipUpdateFlat { + result := make([]protocol.MembershipUpdateFlat, len(chat.MembershipUpdates)) + for idx, update := range chat.MembershipUpdates { + result[idx] = protocol.MembershipUpdateFlat{ + ChatID: chat.ID, + From: update.From, + Signature: update.Signature, + MembershipUpdateEvent: protocol.MembershipUpdateEvent{ + Name: update.Name, + Type: update.Type, + ClockValue: int64(update.ClockValue), // TODO: remove type difference + Member: update.Member, + Members: update.Members, + }, + } + } + return result +} diff --git a/vendor/github.com/status-im/status-protocol-go/crypto/crypto.go b/vendor/github.com/status-im/status-protocol-go/crypto/crypto.go index f54f05b990..d60e0c012d 100644 --- a/vendor/github.com/status-im/status-protocol-go/crypto/crypto.go +++ b/vendor/github.com/status-im/status-protocol-go/crypto/crypto.go @@ -78,6 +78,7 @@ func VerifySignatures(signaturePairs [][3]string) error { } // ExtractSignatures extract from tuples of signatures content a public key +// DEPRECATED: use ExtractSignature func ExtractSignatures(signaturePairs [][2]string) ([]string, error) { response := make([]string, len(signaturePairs)) for i, signaturePair := range signaturePairs { @@ -102,6 +103,12 @@ func ExtractSignatures(signaturePairs [][2]string) ([]string, error) { return response, nil } +// ExtractSignature returns a public key for a given data and signature. +func ExtractSignature(data, signature []byte) (*ecdsa.PublicKey, error) { + dataHash := crypto.Keccak256(data) + return crypto.SigToPub(dataHash, signature) +} + func EncryptSymmetric(key, plaintext []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { diff --git a/vendor/github.com/status-im/status-protocol-go/go.mod b/vendor/github.com/status-im/status-protocol-go/go.mod index af6db7e2ef..108118b745 100644 --- a/vendor/github.com/status-im/status-protocol-go/go.mod +++ b/vendor/github.com/status-im/status-protocol-go/go.mod @@ -16,10 +16,10 @@ require ( github.com/russolsen/same v0.0.0-20160222130632-f089df61f51d // indirect github.com/russolsen/transit v0.0.0-20180705123435-0794b4c4505a github.com/status-im/doubleratchet v2.0.0+incompatible - github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed + github.com/status-im/migrate/v4 v4.6.2-status.2 github.com/status-im/whisper v1.5.1 github.com/stretchr/testify v1.3.1-0.20190712000136-221dbe5ed467 - github.com/vacp2p/mvds v0.0.21 + github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d go.uber.org/zap v1.10.0 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect diff --git a/vendor/github.com/status-im/status-protocol-go/go.sum b/vendor/github.com/status-im/status-protocol-go/go.sum index e5b0c0aaef..02fcb4fbf0 100644 --- a/vendor/github.com/status-im/status-protocol-go/go.sum +++ b/vendor/github.com/status-im/status-protocol-go/go.sum @@ -44,6 +44,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= @@ -68,8 +69,7 @@ github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtf github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.0.0-20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -100,10 +100,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang-migrate/migrate v3.5.4+incompatible h1:R7OzwvCJTCgwapPCiX6DyBiu2czIUMDCB118gFTKTUA= -github.com/golang-migrate/migrate v3.5.4+incompatible/go.mod h1:IsVUlFN5puWOmXrqjgGUfIRIbU7mr8oNBE2tyERd9Wk= -github.com/golang-migrate/migrate/v4 v4.5.0 h1:ucd2qJu1BAKTtmjh7QlWYiq01DwlE/xXYQkOPE0ZTsI= -github.com/golang-migrate/migrate/v4 v4.5.0/go.mod h1:SzAcz2l+yDJVhQC7fwiF7T2MAFPMIkigJz98klRJ4OE= +github.com/golang-migrate/migrate/v4 v4.6.2 h1:LDDOHo/q1W5UDj6PbkxdCv7lv9yunyZHXvxuwDkGo3k= +github.com/golang-migrate/migrate/v4 v4.6.2/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -189,7 +187,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= +github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mutecomm/go-sqlcipher v0.0.0-20190227152316-55dbde17881f h1:hd3r+uv9DNLScbOrnlj82rBldHQf3XWmCeXAWbw8euQ= github.com/mutecomm/go-sqlcipher v0.0.0-20190227152316-55dbde17881f/go.mod h1:MyUWrZlB1aI5bs7j9/pJ8ckLLZ4QcCYcNiSbsAW32D4= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -255,8 +253,8 @@ github.com/status-im/doubleratchet v2.0.0+incompatible h1:s77lF1lDubK0RKftxN2vH8 github.com/status-im/doubleratchet v2.0.0+incompatible/go.mod h1:1sqR0+yhiM/bd+wrdX79AOt2csZuJOni0nUDzKNuqOU= github.com/status-im/go-ethereum v1.9.5-status.4 h1:F5VrxH9LmTxWl4qwQjs0TI5TgG9dVuZKqGmdwHJ0cWk= github.com/status-im/go-ethereum v1.9.5-status.4/go.mod h1:Ulij8LMpMvXnbnPcmDqrpI+iXoXSjxItuY/wmbasTZU= -github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed h1:K2iga8l8OQIHnk2bBq2QsZTO2Q38YWy04xIspdITCdM= -github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed/go.mod h1:r8HggRBZ/k7TRwByq/Hp3P/ubFppIna0nvyavVK0pjA= +github.com/status-im/migrate/v4 v4.6.2-status.2 h1:SdC+sMDl/aI7vUlwD2qj2p7KsK4T60IS9z4/rYCCbI8= +github.com/status-im/migrate/v4 v4.6.2-status.2/go.mod h1:c/kc90n47GZu/58nnz1OMLTf7uE4Da4gZP5qmU+A/v8= github.com/status-im/whisper v1.5.1 h1:87/XIg0Wjua7lXBGiEXgAfTOqlt2Q1dMDuxugTyZbbA= github.com/status-im/whisper v1.5.1/go.mod h1:emrOxzJme0k66QtbbQ2bdd3P8RCdLZ8sTD7SkwH1s2s= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= @@ -276,12 +274,13 @@ github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2K github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/uber/jaeger-client-go v0.0.0-20180607151842-f7e0d4744fa6/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v0.0.0-20180615202729-a51202d6f4a7/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/vacp2p/mvds v0.0.21 h1:YeYja8noKsHvrnOHM4pqjBvDwwy+kUzXMkX1IBYJAbU= -github.com/vacp2p/mvds v0.0.21/go.mod h1:pIqr2Hg4cIkTJniGPCp4ptong2jxgxx6uToVoY94+II= +github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d h1:kc/9/ZbwyqIKQ2phzmPpBYGx7v2olwD2P7Uj8KKd+Bw= +github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d/go.mod h1:uUmtiahU7efOVl/5w5yk9jOze5xYpDZDrSrT8TvHXjQ= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= +go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= diff --git a/vendor/github.com/status-im/status-protocol-go/message_handler.go b/vendor/github.com/status-im/status-protocol-go/message_handler.go new file mode 100644 index 0000000000..734f300fb6 --- /dev/null +++ b/vendor/github.com/status-im/status-protocol-go/message_handler.go @@ -0,0 +1,68 @@ +package statusproto + +import ( + "github.com/pkg/errors" + + protocol "github.com/status-im/status-protocol-go/v1" +) + +type persistentMessageHandler struct { + persistence *sqlitePersistence +} + +func newPersistentMessageHandler(persistence *sqlitePersistence) *persistentMessageHandler { + return &persistentMessageHandler{persistence: persistence} +} + +// HandleMembershipUpdate updates a Chat instance according to the membership updates. +// It retrieves chat, if exists, and merges membership updates from the message. +// Finally, the Chat is updated with the new group events. +func (h *persistentMessageHandler) HandleMembershipUpdate(m protocol.MembershipUpdateMessage) error { + chat, err := h.chatID(m.ChatID) + switch err { + case errChatNotFound: + group, err := protocol.NewGroupWithMembershipUpdates(m.ChatID, m.Updates) + if err != nil { + return err + } + newChat := createGroupChat() + newChat.updateChatFromProtocolGroup(group) + chat = &newChat + case nil: + existingGroup, err := newProtocolGroupFromChat(chat) + if err != nil { + return errors.Wrap(err, "failed to create a Group from Chat") + } + updateGroup, err := protocol.NewGroupWithMembershipUpdates(m.ChatID, m.Updates) + if err != nil { + return errors.Wrap(err, "invalid membership update") + } + merged := protocol.MergeFlatMembershipUpdates(existingGroup.Updates(), updateGroup.Updates()) + newGroup, err := protocol.NewGroup(chat.ID, merged) + if err != nil { + return errors.Wrap(err, "failed to create a group with new membership updates") + } + chat.updateChatFromProtocolGroup(newGroup) + default: + return err + } + return h.persistence.SaveChat(*chat) +} + +func (h *persistentMessageHandler) chatID(chatID string) (*Chat, error) { + var chat *Chat + chats, err := h.persistence.Chats() + if err != nil { + return nil, err + } + for _, ch := range chats { + if chat.ID == chatID { + chat = ch + break + } + } + if chat == nil { + return nil, errChatNotFound + } + return chat, nil +} diff --git a/vendor/github.com/status-im/status-protocol-go/message_processor.go b/vendor/github.com/status-im/status-protocol-go/message_processor.go index bd3cca4131..7c2e2b8d42 100644 --- a/vendor/github.com/status-im/status-protocol-go/message_processor.go +++ b/vendor/github.com/status-im/status-protocol-go/message_processor.go @@ -28,11 +28,16 @@ const ( whisperPoWTime = 5 ) +type messageHandler interface { + HandleMembershipUpdate(m protocol.MembershipUpdateMessage) error +} + type messageProcessor struct { identity *ecdsa.PrivateKey datasync *datasync.DataSync protocol *encryption.Protocol transport *transport.WhisperServiceTransport + handler messageHandler logger *zap.Logger featureFlags featureFlags @@ -43,6 +48,7 @@ func newMessageProcessor( database *sql.DB, enc *encryption.Protocol, transport *transport.WhisperServiceTransport, + handler messageHandler, logger *zap.Logger, features featureFlags, ) (*messageProcessor, error) { @@ -65,6 +71,7 @@ func newMessageProcessor( datasync: ds, protocol: enc, transport: transport, + handler: handler, logger: logger, featureFlags: features, } @@ -87,14 +94,14 @@ func (p *messageProcessor) Stop() { func (p *messageProcessor) SendPrivate( ctx context.Context, - publicKey *ecdsa.PublicKey, + recipient *ecdsa.PublicKey, chatID string, data []byte, clock int64, ) ([]byte, *protocol.Message, error) { p.logger.Debug( "sending a private message", - zap.Binary("public-key", crypto.FromECDSAPub(publicKey)), + zap.Binary("public-key", crypto.FromECDSAPub(recipient)), ) message := protocol.CreatePrivateTextMessage(data, clock, chatID) @@ -103,7 +110,7 @@ func (p *messageProcessor) SendPrivate( return nil, nil, errors.Wrap(err, "failed to encode message") } - messageID, err := p.sendPrivate(ctx, publicKey, encodedMessage) + messageID, err := p.sendPrivate(ctx, recipient, encodedMessage) if err != nil { return nil, nil, err } @@ -113,23 +120,25 @@ func (p *messageProcessor) SendPrivate( // SendPrivateRaw takes encoded data, encrypts it and sends through the wire. func (p *messageProcessor) SendPrivateRaw( ctx context.Context, - publicKey *ecdsa.PublicKey, + recipient *ecdsa.PublicKey, data []byte, ) ([]byte, error) { p.logger.Debug( "sending a private message", - zap.Binary("public-key", crypto.FromECDSAPub(publicKey)), + zap.Binary("public-key", crypto.FromECDSAPub(recipient)), zap.String("site", "SendPrivateRaw"), ) - return p.sendPrivate(ctx, publicKey, data) + return p.sendPrivate(ctx, recipient, data) } // sendPrivate sends data to the recipient identifying with a given public key. func (p *messageProcessor) sendPrivate( ctx context.Context, - publicKey *ecdsa.PublicKey, + recipient *ecdsa.PublicKey, data []byte, ) ([]byte, error) { + p.logger.Debug("sending private message", zap.Binary("recipient", crypto.FromECDSAPub(recipient))) + wrappedMessage, err := p.tryWrapMessageV1(data) if err != nil { return nil, errors.Wrap(err, "failed to wrap message") @@ -138,19 +147,19 @@ func (p *messageProcessor) sendPrivate( messageID := protocol.MessageID(&p.identity.PublicKey, wrappedMessage) if p.featureFlags.datasync { - if err := p.addToDataSync(publicKey, wrappedMessage); err != nil { + if err := p.addToDataSync(recipient, wrappedMessage); err != nil { return nil, errors.Wrap(err, "failed to send message with datasync") } // No need to call transport tracking. // It is done in a data sync dispatch step. } else { - messageSpec, err := p.protocol.BuildDirectMessage(p.identity, publicKey, wrappedMessage) + messageSpec, err := p.protocol.BuildDirectMessage(p.identity, recipient, wrappedMessage) if err != nil { return nil, errors.Wrap(err, "failed to encrypt message") } - hash, newMessage, err := p.sendMessageSpec(ctx, publicKey, messageSpec) + hash, newMessage, err := p.sendMessageSpec(ctx, recipient, messageSpec) if err != nil { return nil, errors.Wrap(err, "failed to send a message spec") } @@ -161,6 +170,61 @@ func (p *messageProcessor) sendPrivate( return messageID, nil } +func (p *messageProcessor) SendGroup( + ctx context.Context, + recipients []*ecdsa.PublicKey, + chatID string, + data []byte, + clock int64, +) ([][]byte, []*protocol.Message, error) { + p.logger.Debug("sending a group message", zap.Int("membersCount", len(recipients))) + + message := protocol.CreatePrivateGroupTextMessage(data, clock, chatID) + encodedMessage, err := p.encodeMessage(message) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to encode message") + } + + var resultIDs [][]byte + for _, recipient := range recipients { + messageID, err := p.sendPrivate(ctx, recipient, encodedMessage) + if err != nil { + return nil, nil, err + } + resultIDs = append(resultIDs, messageID) + } + return resultIDs, nil, nil +} + +func (p *messageProcessor) SendMembershipUpdate( + ctx context.Context, + recipients []*ecdsa.PublicKey, + chatID string, + updates []protocol.MembershipUpdate, + clock int64, +) ([][]byte, error) { + p.logger.Debug("sending a membership update", zap.Int("membersCount", len(recipients))) + + message := protocol.MembershipUpdateMessage{ + ChatID: chatID, + Updates: updates, + } + encodedMessage, err := protocol.EncodeMembershipUpdateMessage(message) + if err != nil { + return nil, errors.Wrap(err, "failed to encode membership update message") + } + + var resultIDs [][]byte + for _, recipient := range recipients { + messageID, err := p.sendPrivate(ctx, recipient, encodedMessage) + if err != nil { + return nil, err + } + resultIDs = append(resultIDs, messageID) + } + return resultIDs, nil +} + func (p *messageProcessor) SendPublic(ctx context.Context, chatID string, data []byte, clock int64) ([]byte, error) { logger := p.logger.With(zap.String("site", "SendPublic")) logger.Debug("sending a public message", zap.String("chatID", chatID)) @@ -249,6 +313,18 @@ func (p *messageProcessor) Process(shhMessage *whispertypes.Message) ([]*protoco m.ID = statusMessage.ID m.SigPubKey = statusMessage.SigPubKey() decodedMessages = append(decodedMessages, &m) + case protocol.MembershipUpdateMessage: + // Handle user message that can be attached to the membership update. + userMessage := m.Message + if userMessage != nil { + userMessage.ID = statusMessage.ID + userMessage.SigPubKey = statusMessage.SigPubKey() + decodedMessages = append(decodedMessages, userMessage) + } + + if err := p.processMembershipUpdate(m); err != nil { + hlogger.Error("failed to process MembershipUpdateMessage", zap.Error(err)) + } case protocol.PairMessage: fromOurDevice := isPubKeyEqual(statusMessage.SigPubKey(), &p.identity.PublicKey) if !fromOurDevice { @@ -267,6 +343,16 @@ func (p *messageProcessor) Process(shhMessage *whispertypes.Message) ([]*protoco return decodedMessages, nil } +func (p *messageProcessor) processMembershipUpdate(m protocol.MembershipUpdateMessage) error { + if err := m.Verify(); err != nil { + return err + } + if p.handler != nil { + return p.handler.HandleMembershipUpdate(m) + } + return errors.New("missing handler") +} + func (p *messageProcessor) processPairMessage(m protocol.PairMessage) error { metadata := &multidevice.InstallationMetadata{ Name: m.Name, diff --git a/vendor/github.com/status-im/status-protocol-go/messenger.go b/vendor/github.com/status-im/status-protocol-go/messenger.go index db8416cd26..3919266a4f 100644 --- a/vendor/github.com/status-im/status-protocol-go/messenger.go +++ b/vendor/github.com/status-im/status-protocol-go/messenger.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/pkg/errors" "go.uber.org/zap" @@ -284,6 +286,7 @@ func NewMessenger( database, encryptionProtocol, t, + newPersistentMessageHandler(&sqlitePersistence{db: database}), logger, c.featureFlags, ) @@ -454,12 +457,20 @@ func (m *Messenger) Mailservers() ([]string, error) { } func (m *Messenger) Join(chat Chat) error { - if chat.PublicKey != nil { + switch chat.ChatType { + case ChatTypeOneToOne: return m.transport.JoinPrivate(chat.PublicKey) - } else if chat.Name != "" { + case ChatTypePrivateGroupChat: + members, err := chat.MembersAsPublicKeys() + if err != nil { + return err + } + return m.transport.JoinGroup(members) + case ChatTypePublic: return m.transport.JoinPublic(chat.Name) + default: + return errors.New("chat is neither public nor private") } - return errors.New("chat is neither public nor private") } func (m *Messenger) Leave(chat Chat) error { @@ -471,6 +482,94 @@ func (m *Messenger) Leave(chat Chat) error { return errors.New("chat is neither public nor private") } +// TODO: consider moving to a ChatManager ??? +func (m *Messenger) CreateGroupChat(name string) (*Chat, error) { + chat := createGroupChat() + group, err := protocol.NewGroupWithCreator(name, m.identity) + if err != nil { + return nil, err + } + chat.updateChatFromProtocolGroup(group) + return &chat, nil +} + +func (m *Messenger) AddMembersToChat(ctx context.Context, chat *Chat, members []*ecdsa.PublicKey) error { + group, err := newProtocolGroupFromChat(chat) + if err != nil { + return err + } + encodedMembers := make([]string, len(members)) + for idx, member := range members { + encodedMembers[idx] = hexutil.Encode(crypto.FromECDSAPub(member)) + } + event := protocol.NewMembersAddedEvent(encodedMembers, group.NextClockValue()) + err = group.ProcessEvent(&m.identity.PublicKey, event) + if err != nil { + return err + } + if err := m.propagateMembershipUpdates(ctx, group); err != nil { + return err + } + chat.updateChatFromProtocolGroup(group) + return m.SaveChat(*chat) +} + +func (m *Messenger) ConfirmJoiningGroup(ctx context.Context, chat *Chat) error { + group, err := newProtocolGroupFromChat(chat) + if err != nil { + return err + } + event := protocol.NewMemberJoinedEvent( + statusproto.EncodeHex(crypto.FromECDSAPub(&m.identity.PublicKey)), + group.NextClockValue(), + ) + err = group.ProcessEvent(&m.identity.PublicKey, event) + if err != nil { + return err + } + if err := m.propagateMembershipUpdates(ctx, group); err != nil { + return err + } + chat.updateChatFromProtocolGroup(group) + return m.SaveChat(*chat) +} + +func (m *Messenger) propagateMembershipUpdates(ctx context.Context, group *protocol.Group) error { + events := make([]protocol.MembershipUpdateEvent, len(group.Updates())) + for idx, event := range group.Updates() { + events[idx] = event.MembershipUpdateEvent + } + update := protocol.MembershipUpdate{ + ChatID: group.ChatID(), + Events: events, + } + if err := update.Sign(m.identity); err != nil { + return err + } + recipients, err := stringSliceToPublicKeys(group.Members(), true) + if err != nil { + return err + } + // Filter out my key from the recipients + n := 0 + for _, recipient := range recipients { + if !isPubKeyEqual(recipient, &m.identity.PublicKey) { + recipients[n] = recipient + n++ + } + } + recipients = recipients[:n] + // Finally send membership updates to all recipients. + _, err = m.processor.SendMembershipUpdate( + ctx, + recipients, + group.ChatID(), + []protocol.MembershipUpdate{update}, + group.NextClockValue(), + ) + return err +} + func (m *Messenger) SaveChat(chat Chat) error { return m.persistence.SaveChat(chat) } @@ -522,7 +621,7 @@ func (m *Messenger) Contacts() ([]*Contact, error) { return m.persistence.Contacts() } -func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([]byte, error) { +func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([][]byte, error) { logger := m.logger.With(zap.String("site", "Send"), zap.String("chatID", chatID)) // A valid added chat is required. @@ -538,7 +637,8 @@ func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([]byt logger.Debug("last message clock received", zap.Int64("clock", clock)) - if chat.PublicKey != nil { + switch chat.ChatType { + case ChatTypeOneToOne: logger.Debug("sending private message", zap.Binary("publicKey", crypto.FromECDSAPub(chat.PublicKey))) id, message, err := m.processor.SendPrivate(ctx, chat.PublicKey, chat.ID, data, clock) @@ -546,27 +646,64 @@ func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([]byt return nil, err } - // Save our message because it won't be received from the transport layer. - message.ID = id // a Message need ID to be properly stored in the db - message.SigPubKey = &m.identity.PublicKey - message.ChatID = chatID + if err := m.cacheOwnMessage(chatID, id, message); err != nil { + return nil, err + } - if m.messagesPersistenceEnabled { - _, err = m.persistence.SaveMessages([]*protocol.Message{message}) - if err != nil { + return [][]byte{id}, nil + case ChatTypePublic: + logger.Debug("sending public message", zap.String("chatName", chat.Name)) + id, err := m.processor.SendPublic(ctx, chat.ID, data, clock) + if err != nil { + return nil, err + } + return [][]byte{id}, nil + case ChatTypePrivateGroupChat: + logger.Debug("sending group message", zap.String("chatName", chat.Name)) + recipients, err := chat.MembersAsPublicKeys() + if err != nil { + return nil, err + } + // Filter me out of recipients. + n := 0 + for _, item := range recipients { + if !isPubKeyEqual(item, &m.identity.PublicKey) { + recipients[n] = item + n++ + } + } + ids, messages, err := m.processor.SendGroup(ctx, recipients[:n], chat.ID, data, clock) + if err != nil { + return nil, err + } + for idx, message := range messages { + if err := m.cacheOwnMessage(chatID, ids[idx], message); err != nil { return nil, err } } + return ids, nil + default: + return nil, errors.New("chat is neither public nor private") + } +} - // Cache it to be returned in Retrieve(). - m.ownMessages = append(m.ownMessages, message) +func (m *Messenger) cacheOwnMessage(chatID string, id []byte, message *protocol.Message) error { + // Save our message because it won't be received from the transport layer. + message.ID = id // a Message need ID to be properly stored in the db + message.SigPubKey = &m.identity.PublicKey + message.ChatID = chatID - return id, nil - } else if chat.Name != "" { - logger.Debug("sending public message", zap.String("chatName", chat.Name)) - return m.processor.SendPublic(ctx, chat.ID, data, clock) + if m.messagesPersistenceEnabled { + _, err := m.persistence.SaveMessages([]*protocol.Message{message}) + if err != nil { + return err + } } - return nil, errors.New("chat is neither public nor private") + + // Cache it to be returned in Retrieve(). + m.ownMessages = append(m.ownMessages, message) + + return nil } // SendRaw takes encoded data, encrypts it and sends through the wire. diff --git a/vendor/github.com/status-im/status-protocol-go/transport/whisper/whisper_service.go b/vendor/github.com/status-im/status-protocol-go/transport/whisper/whisper_service.go index 7214c67641..ef7a0f840f 100644 --- a/vendor/github.com/status-im/status-protocol-go/transport/whisper/whisper_service.go +++ b/vendor/github.com/status-im/status-protocol-go/transport/whisper/whisper_service.go @@ -81,6 +81,9 @@ type WhisperServiceTransport struct { } // NewWhisperServiceTransport returns a new WhisperServiceTransport. +// TODO: leaving a chat should verify that for a given public key +// there are no other chats. It may happen that we leave a private chat +// but still have a public chat for a given public key. func NewWhisperServiceTransport( shh whispertypes.Whisper, privateKey *ecdsa.PrivateKey, @@ -185,6 +188,30 @@ func (a *WhisperServiceTransport) LeavePrivate(publicKey *ecdsa.PublicKey) error return a.filters.Remove(filters...) } +func (a *WhisperServiceTransport) JoinGroup(publicKeys []*ecdsa.PublicKey) error { + _, err := a.filters.LoadDiscovery() + if err != nil { + return err + } + for _, pk := range publicKeys { + _, err = a.filters.LoadContactCode(pk) + if err != nil { + return err + } + } + return nil +} + +func (a *WhisperServiceTransport) LeaveGroup(publicKeys []*ecdsa.PublicKey) error { + for _, publicKey := range publicKeys { + filters := a.filters.FiltersByPublicKey(publicKey) + if err := a.filters.Remove(filters...); err != nil { + return err + } + } + return nil +} + type Message struct { Message *whispertypes.Message Public bool diff --git a/vendor/github.com/status-im/status-protocol-go/v1/decoder.go b/vendor/github.com/status-im/status-protocol-go/v1/decoder.go index ea17a4f832..1ee9d853cd 100644 --- a/vendor/github.com/status-im/status-protocol-go/v1/decoder.go +++ b/vendor/github.com/status-im/status-protocol-go/v1/decoder.go @@ -163,10 +163,6 @@ func membershipUpdateMessageHandler(d transit.Decoder, value interface{}) (inter if !ok { break } - update.From, ok = value[transit.Keyword("from")].(string) - if !ok { - break - } update.Signature, ok = value[transit.Keyword("signature")].(string) if !ok { break diff --git a/vendor/github.com/status-im/status-protocol-go/v1/encoder.go b/vendor/github.com/status-im/status-protocol-go/v1/encoder.go index b0cc5499b6..a9754c4eae 100644 --- a/vendor/github.com/status-im/status-protocol-go/v1/encoder.go +++ b/vendor/github.com/status-im/status-protocol-go/v1/encoder.go @@ -77,7 +77,6 @@ func (messageValueEncoder) Encode(e transit.Encoder, value reflect.Value, asStri element := map[interface{}]interface{}{ transit.Keyword("chat-id"): update.ChatID, - transit.Keyword("from"): update.From, transit.Keyword("events"): events, transit.Keyword("signature"): update.Signature, } diff --git a/vendor/github.com/status-im/status-protocol-go/v1/membership_update_message.go b/vendor/github.com/status-im/status-protocol-go/v1/membership_update_message.go index ac8e2ed993..e0e73a58b2 100644 --- a/vendor/github.com/status-im/status-protocol-go/v1/membership_update_message.go +++ b/vendor/github.com/status-im/status-protocol-go/v1/membership_update_message.go @@ -3,10 +3,18 @@ package statusproto import ( "bytes" "crypto/ecdsa" + "encoding/hex" "encoding/json" + "fmt" "reflect" "sort" + "strings" + "time" + "github.com/ethereum/go-ethereum/common/hexutil" + gethcrypto "github.com/ethereum/go-ethereum/crypto" + "github.com/google/uuid" + "github.com/pkg/errors" "github.com/status-im/status-protocol-go/crypto" ) @@ -29,11 +37,35 @@ type MembershipUpdateMessage struct { Message *Message `json:"message"` // optional message } +// Verify makes sure that the received update message has a valid signature. +// It also extracts public key from the signature available as From field. +// It does not verify the updates and their events. This should be done +// separately using Group struct. +func (m *MembershipUpdateMessage) Verify() error { + for idx, update := range m.Updates { + if err := update.extractFrom(); err != nil { + return errors.Wrapf(err, "failed to extract an author of %d update", idx) + } + m.Updates[idx] = update + } + return nil +} + +// EncodeMembershipUpdateMessage encodes a MembershipUpdateMessage using Transit serialization. +func EncodeMembershipUpdateMessage(value MembershipUpdateMessage) ([]byte, error) { + var buf bytes.Buffer + encoder := NewMessageEncoder(&buf) + if err := encoder.Encode(value); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + type MembershipUpdate struct { ChatID string `json:"chatId"` - From string `json:"from"` - Signature string `json:"signature"` + Signature string `json:"signature"` // hex-encoded without 0x prefix Events []MembershipUpdateEvent `json:"events"` + From string `json:"from"` // hex-encoded with 0x prefix } // Sign creates a signature from MembershipUpdateEvents @@ -41,29 +73,46 @@ type MembershipUpdate struct { // It follows the algorithm describe in the spec: // https://github.com/status-im/specs/blob/master/status-group-chats-spec.md#signature. func (u *MembershipUpdate) Sign(identity *ecdsa.PrivateKey) error { - sort.Slice(u.Events, func(i, j int) bool { - return u.Events[i].ClockValue < u.Events[j].ClockValue - }) - tuples := make([]interface{}, len(u.Events)) - for idx, event := range u.Events { - tuples[idx] = tupleMembershipUpdateEvent(event) + signature, err := createMembershipUpdateSignature(u.ChatID, u.Events, identity) + if err != nil { + return err } - structureToSign := []interface{}{ - tuples, - u.ChatID, + u.Signature = signature + return nil +} + +func (u *MembershipUpdate) extractFrom() error { + content, err := stringifyMembershipUpdateEvents(u.ChatID, u.Events) + if err != nil { + return errors.Wrap(err, "failed to stringify events") } - data, err := json.Marshal(structureToSign) + signatureBytes, err := hex.DecodeString(u.Signature) if err != nil { - return err + return errors.Wrap(err, "failed to decode signature") } - signature, err := crypto.SignBytesAsHex(data, identity) + publicKey, err := crypto.ExtractSignature(content, signatureBytes) if err != nil { - return err + return errors.Wrap(err, "failed to extract signature") } - u.Signature = signature + u.From = hexutil.Encode(gethcrypto.FromECDSAPub(publicKey)) return nil } +func (u *MembershipUpdate) Flat() []MembershipUpdateFlat { + result := make([]MembershipUpdateFlat, 0, len(u.Events)) + for _, event := range u.Events { + result = append(result, MembershipUpdateFlat{ + MembershipUpdateEvent: event, + ChatID: u.ChatID, + Signature: u.Signature, + From: u.From, + }) + } + return result +} + +// MembershipUpdateEvent contains an event information. +// Member and Members are hex-encoded values with 0x prefix. type MembershipUpdateEvent struct { Type string `json:"type"` ClockValue int64 `json:"clockValue"` @@ -72,10 +121,49 @@ type MembershipUpdateEvent struct { Name string `json:"name,omitempty"` // name of the group chat } -func NewChatCreatedEvent(name string, clock int64) MembershipUpdateEvent { +func (u MembershipUpdateEvent) Equal(update MembershipUpdateEvent) bool { + return u.Type == update.Type && + u.ClockValue == update.ClockValue && + u.Member == update.Member && + stringSliceEquals(u.Members, update.Members) && + u.Name == update.Name +} + +type MembershipUpdateFlat struct { + MembershipUpdateEvent + ChatID string `json:"chatId"` + Signature string `json:"signature"` + From string `json:"from"` +} + +func (u MembershipUpdateFlat) Equal(update MembershipUpdateFlat) bool { + return u.ChatID == update.ChatID && + u.Signature == update.Signature && + u.From == update.From && + u.MembershipUpdateEvent.Equal(update.MembershipUpdateEvent) +} + +func MergeFlatMembershipUpdates(dest []MembershipUpdateFlat, src []MembershipUpdateFlat) []MembershipUpdateFlat { + for _, update := range src { + var exists bool + for _, existing := range dest { + if existing.Equal(update) { + exists = true + break + } + } + if !exists { + dest = append(dest, update) + } + } + return dest +} + +func NewChatCreatedEvent(name string, admin string, clock int64) MembershipUpdateEvent { return MembershipUpdateEvent{ Type: MembershipUpdateChatCreated, Name: name, + Member: admin, ClockValue: clock, } } @@ -128,14 +216,27 @@ func NewAdminRemovedEvent(admin string, clock int64) MembershipUpdateEvent { } } -// EncodeMembershipUpdateMessage encodes a MembershipUpdateMessage using Transit serialization. -func EncodeMembershipUpdateMessage(value MembershipUpdateMessage) ([]byte, error) { - var buf bytes.Buffer - encoder := NewMessageEncoder(&buf) - if err := encoder.Encode(value); err != nil { - return nil, err +func stringifyMembershipUpdateEvents(chatID string, events []MembershipUpdateEvent) ([]byte, error) { + sort.Slice(events, func(i, j int) bool { + return events[i].ClockValue < events[j].ClockValue + }) + tuples := make([]interface{}, len(events)) + for idx, event := range events { + tuples[idx] = tupleMembershipUpdateEvent(event) } - return buf.Bytes(), nil + structureToSign := []interface{}{ + tuples, + chatID, + } + return json.Marshal(structureToSign) +} + +func createMembershipUpdateSignature(chatID string, events []MembershipUpdateEvent, identity *ecdsa.PrivateKey) (string, error) { + data, err := stringifyMembershipUpdateEvents(chatID, events) + if err != nil { + return "", err + } + return crypto.SignBytesAsHex(data, identity) } var membershipUpdateEventFieldNamesCompat = map[string]string{ @@ -173,41 +274,222 @@ func tupleMembershipUpdateEvent(update MembershipUpdateEvent) [][]interface{} { } type Group struct { - ChatID string - Admins []string - Contacts []string + chatID string + name string + updates []MembershipUpdateFlat + admins *stringSet + members *stringSet +} + +func groupChatID(creator *ecdsa.PublicKey) string { + return uuid.New().String() + "-" + hexutil.Encode(gethcrypto.FromECDSAPub(creator)) +} + +func NewGroupWithMembershipUpdates(chatID string, updates []MembershipUpdate) (*Group, error) { + flatten := make([]MembershipUpdateFlat, 0, len(updates)) + for _, update := range updates { + flatten = append(flatten, update.Flat()...) + } + return newGroup(chatID, flatten) +} + +func NewGroupWithCreator(name string, creator *ecdsa.PrivateKey) (*Group, error) { + chatID := groupChatID(&creator.PublicKey) + creatorHex := publicKeyToString(&creator.PublicKey) + clock := TimestampInMsFromTime(time.Now()) + chatCreated := NewChatCreatedEvent(name, creatorHex, int64(clock)) + update := MembershipUpdate{ + ChatID: chatID, + From: creatorHex, + Events: []MembershipUpdateEvent{chatCreated}, + } + if err := update.Sign(creator); err != nil { + return nil, err + } + return newGroup(chatID, update.Flat()) +} + +func NewGroup(chatID string, updates []MembershipUpdateFlat) (*Group, error) { + return newGroup(chatID, updates) +} + +func newGroup(chatID string, updates []MembershipUpdateFlat) (*Group, error) { + g := Group{ + chatID: chatID, + updates: updates, + admins: newStringSet(), + members: newStringSet(), + } + if err := g.init(); err != nil { + return nil, err + } + return &g, nil } -// ValidateEvent returns true if a given event is valid. -func (g *Group) ValidateEvent(from string, event MembershipUpdateEvent) bool { +func (g *Group) init() error { + g.sortEvents() + + var chatID string + + for _, update := range g.updates { + if chatID == "" { + chatID = update.ChatID + } else if update.ChatID != chatID { + return errors.New("updates contain different chat IDs") + } + valid := g.validateEvent(update.From, update.MembershipUpdateEvent) + if !valid { + return fmt.Errorf("invalid event %#+v from %s", update.MembershipUpdateEvent, update.From) + } + g.processEvent(update.From, update.MembershipUpdateEvent) + } + + valid := g.validateChatID(g.chatID) + if !valid { + return fmt.Errorf("invalid chat ID: %s", g.chatID) + } + if chatID != g.chatID { + return fmt.Errorf("expected chat ID equal %s, got %s", g.chatID, chatID) + } + + return nil +} + +func (g Group) ChatID() string { + return g.chatID +} + +func (g Group) Updates() []MembershipUpdateFlat { + return g.updates +} + +func (g Group) Name() string { + return g.name +} + +func (g Group) Members() []string { + return g.members.List() +} + +func (g Group) Admins() []string { + return g.admins.List() +} + +func (g Group) Joined() []string { + var result []string + for _, update := range g.updates { + if update.Type == MembershipUpdateMemberJoined { + result = append(result, update.Member) + } + } + return result +} + +func (g *Group) ProcessEvents(from *ecdsa.PublicKey, events []MembershipUpdateEvent) error { + for _, event := range events { + err := g.ProcessEvent(from, event) + if err != nil { + return err + } + } + return nil +} + +func (g *Group) ProcessEvent(from *ecdsa.PublicKey, event MembershipUpdateEvent) error { + fromHex := hexutil.Encode(gethcrypto.FromECDSAPub(from)) + if !g.validateEvent(fromHex, event) { + return fmt.Errorf("invalid event %#+v from %s", event, from) + } + update := MembershipUpdate{ + ChatID: g.chatID, + From: fromHex, + Events: []MembershipUpdateEvent{event}, + } + g.updates = append(g.updates, update.Flat()...) + g.processEvent(fromHex, event) + return nil +} + +func (g Group) LastClockValue() int64 { + if len(g.updates) == 0 { + return 0 + } + return g.updates[len(g.updates)-1].ClockValue +} + +func (g Group) NextClockValue() int64 { + return g.LastClockValue() + 1 +} + +func (g Group) creator() (string, error) { + if len(g.updates) == 0 { + return "", errors.New("no events in the group") + } + first := g.updates[0] + if first.Type != MembershipUpdateChatCreated { + return "", fmt.Errorf("expected first event to be 'chat-created', got %s", first.Type) + } + return first.From, nil +} + +func (g Group) validateChatID(chatID string) bool { + creator, err := g.creator() + if err != nil || creator == "" { + return false + } + // TODO: It does not verify that the prefix is a valid UUID. + // Improve it so that the prefix follows UUIDv4 spec. + return strings.HasSuffix(chatID, creator) && chatID != creator +} + +// validateEvent returns true if a given event is valid. +func (g Group) validateEvent(from string, event MembershipUpdateEvent) bool { switch event.Type { case MembershipUpdateChatCreated: - return len(g.Admins) == 0 && len(g.Contacts) == 0 + return g.admins.Empty() && g.members.Empty() case MembershipUpdateNameChanged: - return stringSliceContains(g.Admins, from) && len(event.Name) > 0 + return g.admins.Has(from) && len(event.Name) > 0 case MembershipUpdateMembersAdded: - return stringSliceContains(g.Admins, from) + return g.admins.Has(from) case MembershipUpdateMemberJoined: - return stringSliceContains(g.Contacts, from) && from == event.Member + return g.members.Has(from) && from == event.Member case MembershipUpdateMemberRemoved: // Member can remove themselves or admin can remove a member. - return from == event.Member || (stringSliceContains(g.Admins, from) && !stringSliceContains(g.Admins, event.Member)) + return from == event.Member || (g.admins.Has(from) && !g.admins.Has(event.Member)) case MembershipUpdateAdminsAdded: - return stringSliceContains(g.Admins, from) && stringSliceSubset(event.Members, g.Contacts) + return g.admins.Has(from) && stringSliceSubset(event.Members, g.members.List()) case MembershipUpdateAdminRemoved: - return stringSliceContains(g.Admins, from) && from == event.Member + return g.admins.Has(from) && from == event.Member default: return false } } -func stringSliceContains(slice []string, item string) bool { - for _, s := range slice { - if s == item { - return true - } +func (g *Group) processEvent(from string, event MembershipUpdateEvent) { + switch event.Type { + case MembershipUpdateChatCreated: + g.name = event.Name + g.members.Add(event.Member) + g.admins.Add(event.Member) + case MembershipUpdateNameChanged: + g.name = event.Name + case MembershipUpdateAdminsAdded: + g.admins.Add(event.Members...) + case MembershipUpdateAdminRemoved: + g.admins.Remove(event.Member) + case MembershipUpdateMembersAdded: + g.members.Add(event.Members...) + case MembershipUpdateMemberRemoved: + g.members.Remove(event.Member) + case MembershipUpdateMemberJoined: + g.members.Add(event.Member) } - return false +} + +func (g *Group) sortEvents() { + sort.Slice(g.updates, func(i, j int) bool { + return g.updates[i].ClockValue < g.updates[j].ClockValue + }) } func stringSliceSubset(subset []string, set []string) bool { @@ -225,3 +507,82 @@ func stringSliceSubset(subset []string, set []string) bool { } return false } + +func stringSliceEquals(slice1, slice2 []string) bool { + set := map[string]struct{}{} + for _, s := range slice1 { + set[s] = struct{}{} + } + for _, s := range slice2 { + _, ok := set[s] + if !ok { + return false + } + } + return true +} + +func publicKeyToString(publicKey *ecdsa.PublicKey) string { + return hexutil.Encode(gethcrypto.FromECDSAPub(publicKey)) +} + +type stringSet struct { + m map[string]struct{} + items []string +} + +func newStringSet() *stringSet { + return &stringSet{ + m: make(map[string]struct{}), + } +} + +func newStringSetFromSlice(s []string) *stringSet { + set := newStringSet() + if len(s) > 0 { + set.Add(s...) + } + return set +} + +func (s *stringSet) Add(items ...string) { + for _, item := range items { + if _, ok := s.m[item]; !ok { + s.m[item] = struct{}{} + s.items = append(s.items, item) + } + } +} + +func (s *stringSet) Remove(items ...string) { + for _, item := range items { + if _, ok := s.m[item]; ok { + delete(s.m, item) + s.removeFromItems(item) + } + } +} + +func (s *stringSet) Has(item string) bool { + _, ok := s.m[item] + return ok +} + +func (s *stringSet) Empty() bool { + return len(s.items) == 0 +} + +func (s *stringSet) List() []string { + return s.items +} + +func (s *stringSet) removeFromItems(dropped string) { + n := 0 + for _, item := range s.items { + if item != dropped { + s.items[n] = item + n++ + } + } + s.items = s.items[:n] +} diff --git a/vendor/github.com/vacp2p/mvds/node/node.go b/vendor/github.com/vacp2p/mvds/node/node.go index 400e694c03..20d7595088 100644 --- a/vendor/github.com/vacp2p/mvds/node/node.go +++ b/vendor/github.com/vacp2p/mvds/node/node.go @@ -492,7 +492,7 @@ func (n *Node) onMessage(sender state.PeerID, msg protobuf.Message) error { ) err := n.syncState.Remove(id, sender) - if err != nil { + if err != nil && err != state.ErrStateNotFound { return err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 93f3961128..8d0a66041d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -122,9 +122,10 @@ github.com/go-stack/stack # github.com/gogo/protobuf v1.3.0 github.com/gogo/protobuf/io github.com/gogo/protobuf/proto -# github.com/golang-migrate/migrate/v4 v4.5.0 +# github.com/golang-migrate/migrate/v4 v4.6.2 github.com/golang-migrate/migrate/v4 github.com/golang-migrate/migrate/v4/database +github.com/golang-migrate/migrate/v4/internal/url github.com/golang-migrate/migrate/v4/source # github.com/golang/mock v1.3.1 github.com/golang/mock/gomock @@ -339,16 +340,17 @@ github.com/status-im/doubleratchet github.com/status-im/go-multiaddr-ethv4 # github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 github.com/status-im/keycard-go/derivationpath -# github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7 +# github.com/status-im/migrate/v4 v4.6.2-status.2 github.com/status-im/migrate/v4 github.com/status-im/migrate/v4/database/postgres github.com/status-im/migrate/v4/database/sqlcipher +github.com/status-im/migrate/v4/internal/url github.com/status-im/migrate/v4/source/go_bindata # github.com/status-im/rendezvous v1.3.0 github.com/status-im/rendezvous github.com/status-im/rendezvous/protocol github.com/status-im/rendezvous/server -# github.com/status-im/status-protocol-go v0.3.1 +# github.com/status-im/status-protocol-go v0.4.1-0.20191014103017-1f15a058b66c github.com/status-im/status-protocol-go github.com/status-im/status-protocol-go/applicationmetadata github.com/status-im/status-protocol-go/bridge/geth @@ -396,7 +398,7 @@ github.com/syndtr/goleveldb/leveldb/util # github.com/tyler-smith/go-bip39 v1.0.2 github.com/tyler-smith/go-bip39 github.com/tyler-smith/go-bip39/wordlists -# github.com/vacp2p/mvds v0.0.21 +# github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d github.com/vacp2p/mvds/node github.com/vacp2p/mvds/node/migrations github.com/vacp2p/mvds/peers