From cd66baa343228d0c7a65a492c050d0e5b96084a7 Mon Sep 17 00:00:00 2001 From: Sergey Stepanov Date: Mon, 13 Nov 2023 18:46:41 +0300 Subject: [PATCH] Add a generic S3 provider for cloud saves --- go.mod | 29 ++++-- go.sum | 67 +++++++++----- pkg/config/config.yaml | 13 ++- pkg/config/worker.go | 7 +- pkg/worker/caged/libretro/caged.go | 16 ++-- pkg/worker/caged/libretro/cloud.go | 44 +++++---- pkg/worker/caged/libretro/frontend.go | 2 + pkg/worker/cloud/cloudstore.go | 128 -------------------------- pkg/worker/cloud/cloudstore_test.go | 54 ----------- pkg/worker/cloud/s3.go | 91 ++++++++++++++++++ pkg/worker/cloud/s3_test.go | 55 +++++++++++ pkg/worker/cloud/store.go | 24 +++++ pkg/worker/worker.go | 2 +- 13 files changed, 288 insertions(+), 244 deletions(-) delete mode 100644 pkg/worker/cloud/cloudstore.go delete mode 100644 pkg/worker/cloud/cloudstore_test.go create mode 100644 pkg/worker/cloud/s3.go create mode 100644 pkg/worker/cloud/s3_test.go create mode 100644 pkg/worker/cloud/store.go diff --git a/go.mod b/go.mod index e11342a39..df9b85940 100644 --- a/go.mod +++ b/go.mod @@ -11,44 +11,55 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/knadh/koanf/maps v0.1.1 github.com/knadh/koanf/v2 v2.0.1 + github.com/minio/minio-go/v7 v7.0.63 github.com/pion/ice/v3 v3.0.2 - github.com/pion/interceptor v0.1.24 + github.com/pion/interceptor v0.1.25 github.com/pion/logging v0.2.2 - github.com/pion/webrtc/v4 v4.0.0-beta.6 + github.com/pion/webrtc/v4 v4.0.0-beta.7 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.31.0 github.com/veandco/go-sdl2 v0.4.35 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.15.0 golang.org/x/image v0.13.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pion/datachannel v1.5.5 // indirect github.com/pion/dtls/v2 v2.2.7 // indirect github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.10 // indirect - github.com/pion/rtp v1.8.2 // indirect + github.com/pion/rtcp v1.2.12 // indirect + github.com/pion/rtp v1.8.3 // indirect github.com/pion/sctp v1.8.9 // indirect github.com/pion/sdp/v3 v3.0.6 // indirect - github.com/pion/srtp/v3 v3.0.0 // indirect + github.com/pion/srtp/v3 v3.0.1 // indirect github.com/pion/stun/v2 v2.0.0 // indirect github.com/pion/transport/v2 v2.2.4 // indirect github.com/pion/transport/v3 v3.0.1 // indirect github.com/pion/turn/v3 v3.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/stretchr/testify v1.8.4 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 1eb82e824..fa96e0725 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -29,12 +31,20 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= @@ -50,12 +60,23 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -69,30 +90,29 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v3 v3.0.1/go.mod h1:j4tfTlj4aSEQN9gP3IdliSHcUTWTu9tlOZL0c59MFXo= github.com/pion/ice/v3 v3.0.2 h1:dNQnKsjLvOWz+PaI4tw1VnLYTp9adihC1HIASFGajmI= github.com/pion/ice/v3 v3.0.2/go.mod h1:q3BDzTsxbqP0ySMSHrFuw2MYGUx/AC3WQfRGC5F/0Is= -github.com/pion/interceptor v0.1.24 h1:lN4ua3yUAJCgNKQKcZIM52wFjBgjN0r7shLj91PkJ0c= -github.com/pion/interceptor v0.1.24/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= +github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= +github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtp v1.8.1/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.2 h1:oKMM0K1/QYQ5b5qH+ikqDSZRipP5mIxPJcgcvw5sH0w= +github.com/pion/rtcp v1.2.12 h1:bKWiX93XKgDZENEXCijvHRU/wRifm6JV5DGcH6twtSM= +github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= +github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g= github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v3 v3.0.0 h1:dH5nZUTxN+JDu4otle8Dfh5E/MHR6m8/aib7eD22QDc= -github.com/pion/srtp/v3 v3.0.0/go.mod h1:WxJGk0scShe0UdUidDgR0kDHywX7JN83JOYPkYiLdpM= +github.com/pion/srtp/v3 v3.0.1 h1:AkIQRIZ+3tAOJMQ7G301xtrD1vekQbNeRO7eY1K8ZHk= +github.com/pion/srtp/v3 v3.0.1/go.mod h1:3R3a1qIOIxBkVTLGFjafKK6/fJoTdQDhcC67HOyMbJ8= github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= @@ -104,8 +124,8 @@ github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouAN github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/turn/v3 v3.0.1 h1:wLi7BTQr6/Q20R0vt/lHbjv6y4GChFtC33nkYbasoT8= github.com/pion/turn/v3 v3.0.1/go.mod h1:MrJDKgqryDyWy1/4NT9TWfXWGMC7UHT6pJIv1+gMeNE= -github.com/pion/webrtc/v4 v4.0.0-beta.6 h1:swTwlzDY+1zDtW7ogXjNwlUY0xW733UUIAUMNUTCkPw= -github.com/pion/webrtc/v4 v4.0.0-beta.6/go.mod h1:UcyD8jIeTkFqfYJqoHT9qwUSmrtacKaXxgOEujOdhZ8= +github.com/pion/webrtc/v4 v4.0.0-beta.7 h1:OGCl69njLUKzT0ozJEon18W1LqH0GtuxG9Qx+qtxBdg= +github.com/pion/webrtc/v4 v4.0.0-beta.7/go.mod h1:/zWz+1e1qrjaIKYZG/mOfPrntiHOhnd3vGz2Fdo85Ys= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -114,10 +134,14 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -137,11 +161,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -159,12 +183,11 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -183,6 +206,7 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -191,18 +215,17 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= @@ -213,10 +236,10 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -238,6 +261,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/config/config.yaml b/pkg/config/config.yaml index 13104af55..117cfbbbd 100644 --- a/pkg/config/config.yaml +++ b/pkg/config/config.yaml @@ -302,14 +302,19 @@ recording: # save directory folder: ./recording +# cloud storage options +# it is mandatory to use a cloud storage when running +# a distributed multi-server configuration in order to +# share save states between nodes (resume games on a different worker) storage: # cloud storage provider: # - empty (No op storage stub) - # - oracle [Oracle Object Storage](https://www.oracle.com/cloud/storage/object-storage.html) + # - s3 (S3 API compatible object storage) provider: - # this value contains arbitrary key attribute: - # - oracle: pre-authenticated URL (see: https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingpreauthenticatedrequests.htm) - key: + s3Endpoint: + s3BucketName: + s3AccessKeyId: + s3SecretAccessKey: webrtc: # turn off default Pion interceptors (see: https://github.com/pion/interceptor) diff --git a/pkg/config/worker.go b/pkg/config/worker.go index c0c39adf3..5fd4ef23a 100644 --- a/pkg/config/worker.go +++ b/pkg/config/worker.go @@ -22,8 +22,11 @@ type WorkerConfig struct { } type Storage struct { - Provider string - Key string + Provider string + S3Endpoint string + S3BucketName string + S3AccessKeyId string + S3SecretAccessKey string } type Worker struct { diff --git a/pkg/worker/caged/libretro/caged.go b/pkg/worker/caged/libretro/caged.go index f71017bfa..a0b5f6334 100644 --- a/pkg/worker/caged/libretro/caged.go +++ b/pkg/worker/caged/libretro/caged.go @@ -64,14 +64,14 @@ func (c *Caged) EnableRecording(nowait bool, user string, game string) { } func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) { - if storage != nil { - wc, err := WithCloud(c.Emulator, uid, storage) - if err != nil { - c.log.Error().Err(err).Msgf("couldn't init %v", wc.HashPath()) - } else { - c.log.Info().Msgf("cloud state %v has been initialized", wc.HashPath()) - c.Emulator = wc - } + if storage == nil { + return + } + if wc, err := WithCloud(c.Emulator, uid, storage); err == nil { + c.Emulator = wc + c.log.Info().Msgf("cloud storage has been initialized") + } else { + c.log.Error().Err(err).Msgf("couldn't init cloud storage") } } diff --git a/pkg/worker/caged/libretro/cloud.go b/pkg/worker/caged/libretro/cloud.go index caaa03353..67f8da14e 100644 --- a/pkg/worker/caged/libretro/cloud.go +++ b/pkg/worker/caged/libretro/cloud.go @@ -7,32 +7,37 @@ import ( type CloudFrontend struct { Emulator - stateName string - stateLocalPath string - storage cloud.Storage // a cloud storage to store room state online + uid string + storage cloud.Storage // a cloud storage to store room state online } -func WithCloud(fe Emulator, stateName string, storage cloud.Storage) (*CloudFrontend, error) { - r := &CloudFrontend{Emulator: fe, stateLocalPath: fe.HashPath(), stateName: stateName, storage: storage} +// WithCloud adds the ability to keep game states in the cloud storage like Amazon S3. +// It supports only one file of main save state. +func WithCloud(fe Emulator, uid string, storage cloud.Storage) (*CloudFrontend, error) { + r := &CloudFrontend{Emulator: fe, uid: uid, storage: storage} - // saveOnlineRoomToLocal save online room to local. - // !Supports only one file of main save state. - data, err := r.storage.Load(stateName) - if err != nil { - return nil, err - } - // save the data fetched from the cloud to a local directory - if data != nil { - if err := os.WriteFile(r.stateLocalPath, data, 0644); err != nil { + name := fe.SaveStateName() + + if r.storage.Has(name) { + data, err := r.storage.Load(fe.SaveStateName()) + if err != nil { return nil, err } + // save the data fetched from the cloud to a local directory + if data != nil { + if err := os.WriteFile(fe.HashPath(), data, 0644); err != nil { + return nil, err + } + } } return r, nil } +// !to use emulator save/load calls instead of the storage + func (c *CloudFrontend) HasSave() bool { - _, err := c.storage.Load(c.stateName) + _, err := c.storage.Load(c.SaveStateName()) if err == nil { return true } @@ -43,8 +48,13 @@ func (c *CloudFrontend) SaveGameState() error { if err := c.Emulator.SaveGameState(); err != nil { return err } - if err := c.storage.Save(c.stateName, c.stateLocalPath); err != nil { + path := c.Emulator.HashPath() + data, err := os.ReadFile(path) + if err != nil { return err } - return nil + return c.storage.Save(c.SaveStateName(), data, map[string]string{ + "uid": c.uid, + "type": "cloudretro-main-save", + }) } diff --git a/pkg/worker/caged/libretro/frontend.go b/pkg/worker/caged/libretro/frontend.go index 6a729cb3c..dc8316d50 100644 --- a/pkg/worker/caged/libretro/frontend.go +++ b/pkg/worker/caged/libretro/frontend.go @@ -36,6 +36,7 @@ type Emulator interface { // SetSessionId sets distinct name for the game session (in order to save/load it later) SetSessionId(name string) SaveGameState() error + SaveStateName() string // HashPath returns the path emulator will save state to HashPath() string // HasSave returns true if the current ROM was saved before @@ -295,6 +296,7 @@ func (f *Frontend) RestoreGameState() error { return f.Load() } func (f *Frontend) Rotation() uint { return f.nano.Rot } func (f *Frontend) SRAMPath() string { return f.storage.GetSRAMPath() } func (f *Frontend) SaveGameState() error { return f.Save() } +func (f *Frontend) SaveStateName() string { return filepath.Base(f.HashPath()) } func (f *Frontend) Scale() float64 { return f.scale } func (f *Frontend) SetAudioCb(cb func(app.Audio)) { f.onAudio = cb } func (f *Frontend) SetSessionId(name string) { f.storage.SetMainSaveName(name) } diff --git a/pkg/worker/cloud/cloudstore.go b/pkg/worker/cloud/cloudstore.go deleted file mode 100644 index 2413a7a56..000000000 --- a/pkg/worker/cloud/cloudstore.go +++ /dev/null @@ -1,128 +0,0 @@ -package cloud - -import ( - "bytes" - "crypto/md5" - "encoding/base64" - "errors" - "fmt" - "io" - "net/http" - "time" - - "github.com/giongto35/cloud-game/v3/pkg/os" -) - -// !to replace all with unified s3 api - -type Storage interface { - Save(name string, localPath string) (err error) - Load(name string) (data []byte, err error) -} - -type OracleDataStorageClient struct { - accessURL string - client *http.Client -} - -func Store(provider, key string) (Storage, error) { - var st Storage - var err error - switch provider { - case "oracle": - st, err = NewOracleDataStorageClient(key) - case "coordinator": - default: - } - return st, err -} - -// NewOracleDataStorageClient returns either a new Oracle Data Storage -// client or some error in case of failure. -// Oracle infrastructure access is based on pre-authenticated requests, -// see: https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingpreauthenticatedrequests.htm -// -// It follows broken Google Cloud Storage client design. -func NewOracleDataStorageClient(accessURL string) (*OracleDataStorageClient, error) { - if accessURL == "" { - return nil, errors.New("pre-authenticated request was not specified") - } - return &OracleDataStorageClient{ - accessURL: accessURL, - client: &http.Client{ - Timeout: 10 * time.Second, - }, - }, nil -} - -func (s *OracleDataStorageClient) Save(name string, localPath string) (err error) { - if s == nil { - return nil - } - - dat, err := os.ReadFile(localPath) - if err != nil { - return err - } - - req, err := http.NewRequest("PUT", s.accessURL+name, bytes.NewBuffer(dat)) - if err != nil { - return err - } - - resp, err := s.client.Do(req) - if err != nil { - return err - } - defer func() { - _ = resp.Body.Close() - }() - if resp.StatusCode != 200 { - return errors.New(resp.Status) - } - - dstMD5 := resp.Header.Get("Opc-Content-Md5") - srcMD5 := base64.StdEncoding.EncodeToString(md5Hash(dat)) - if dstMD5 != srcMD5 { - return fmt.Errorf("MD5 mismatch %v != %v", srcMD5, dstMD5) - } - - return nil -} - -func (s *OracleDataStorageClient) Load(name string) (data []byte, err error) { - if s == nil { - return nil, errors.New("cloud storage was not initialized") - } - - res, err := s.client.Get(s.accessURL + name) - if err != nil { - return nil, err - } - defer func() { - _ = res.Body.Close() - }() - - if res.StatusCode != 200 { - return nil, errors.New(res.Status) - } - - dat, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - - dstMD5 := res.Header.Get("Content-Md5") - srcMD5 := base64.StdEncoding.EncodeToString(md5Hash(dat)) - if dstMD5 != srcMD5 { - return nil, fmt.Errorf("MD5 mismatch %v != %v", srcMD5, dstMD5) - } - - return dat, nil -} - -func md5Hash(data []byte) []byte { - hash := md5.New() - hash.Write(data) - return hash.Sum(nil) -} diff --git a/pkg/worker/cloud/cloudstore_test.go b/pkg/worker/cloud/cloudstore_test.go deleted file mode 100644 index 228d4d297..000000000 --- a/pkg/worker/cloud/cloudstore_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package cloud - -import ( - "io" - "net/http" - "os" - "strings" - "testing" -) - -type rtFunc func(req *http.Request) *http.Response - -func (f rtFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req), nil } - -func newTestClient(fn rtFunc) *http.Client { - return &http.Client{ - Transport: fn, - } -} - -func TestOracleSave(t *testing.T) { - client, _ := NewOracleDataStorageClient("test-url/") - client.client = newTestClient(func(req *http.Request) *http.Response { - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader("")), - Header: map[string][]string{ - "Opc-Content-Md5": {"CY9rzUYh03PK3k6DJie09g=="}, - }, - } - }) - - tempFile, err := os.CreateTemp("", "oracle_test.file") - if err != nil { - t.Errorf("%v", err) - } - defer func() { - _ = tempFile.Close() - err := os.Remove(tempFile.Name()) - if err != nil { - t.Errorf("%v", err) - } - }() - - _, err = tempFile.WriteString("test") - if err != nil { - return - } - - err = client.Save("oracle_test.file", tempFile.Name()) - if err != nil { - t.Errorf("can't save, err: %v", err) - } -} diff --git a/pkg/worker/cloud/s3.go b/pkg/worker/cloud/s3.go new file mode 100644 index 000000000..bc5227f7e --- /dev/null +++ b/pkg/worker/cloud/s3.go @@ -0,0 +1,91 @@ +package cloud + +import ( + "bytes" + "context" + "errors" + "io" + + "github.com/giongto35/cloud-game/v3/pkg/logger" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rs/zerolog/log" +) + +type S3Client struct { + c *minio.Client + bucket string + log *logger.Logger +} + +func NewS3Client(endpoint, bucket, key, secret string, log *logger.Logger) (*S3Client, error) { + s3Client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(key, secret, ""), + Secure: true, + }) + if err != nil { + return nil, err + } + + exists, err := s3Client.BucketExists(context.Background(), bucket) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.New("bucket doesn't exist") + } + + return &S3Client{bucket: bucket, c: s3Client, log: log}, nil +} + +func (s *S3Client) SetBucket(bucket string) { s.bucket = bucket } + +func (s *S3Client) Save(name string, data []byte, meta map[string]string) error { + if s == nil || s.c == nil { + return errors.New("s3 client was not initialised") + } + r := bytes.NewReader(data) + opts := minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + } + if meta != nil { + opts.UserMetadata = meta + } + + info, err := s.c.PutObject(context.Background(), s.bucket, name, r, int64(len(data)), opts) + if err != nil { + return err + } + s.log.Debug().Msgf("Uploaded: %v", info) + return nil +} + +func (s *S3Client) Load(name string) (data []byte, err error) { + if s == nil || s.c == nil { + return nil, errors.New("s3 client was not initialised") + } + + r, err := s.c.GetObject(context.Background(), s.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer func() { err = errors.Join(err, r.Close()) }() + + stats, err := r.Stat() + log.Debug().Msgf("Downloaded: %v", stats) + dat, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return dat, nil +} + +func (s *S3Client) Has(name string) bool { + if s == nil || s.c == nil { + return false + } + _, err := s.c.StatObject(context.Background(), s.bucket, name, minio.GetObjectOptions{}) + return err == nil +} diff --git a/pkg/worker/cloud/s3_test.go b/pkg/worker/cloud/s3_test.go new file mode 100644 index 000000000..9701cd9c6 --- /dev/null +++ b/pkg/worker/cloud/s3_test.go @@ -0,0 +1,55 @@ +package cloud + +import ( + "crypto/rand" + "testing" + + "github.com/giongto35/cloud-game/v3/pkg/logger" +) + +func TestS3(t *testing.T) { + t.Skip() + + name := "test" + s3, err := NewS3Client( + "s3.tebi.io", + "cloudretro-001", + "", + "", + logger.Default(), + ) + if err != nil { + t.Error(err) + } + + buf := make([]byte, 1024*4) + // then we can call rand.Read. + _, err = rand.Read(buf) + if err != nil { + t.Error(err) + } + + err = s3.Save(name, buf, map[string]string{"id": "test"}) + if err != nil { + t.Error(err) + } + + exists := s3.Has(name) + if !exists { + t.Errorf("don't exist, but shuld") + } + + ne := s3.Has(name + "123213") + if ne { + t.Errorf("exists, but shouldn't") + } + + dat, err := s3.Load(name) + if err != nil { + t.Error(err) + } + + if len(dat) == 0 { + t.Errorf("should be something") + } +} diff --git a/pkg/worker/cloud/store.go b/pkg/worker/cloud/store.go new file mode 100644 index 000000000..538983cf9 --- /dev/null +++ b/pkg/worker/cloud/store.go @@ -0,0 +1,24 @@ +package cloud + +import ( + "github.com/giongto35/cloud-game/v3/pkg/config" + "github.com/giongto35/cloud-game/v3/pkg/logger" +) + +type Storage interface { + Save(name string, data []byte, tags map[string]string) (err error) + Load(name string) (data []byte, err error) + Has(name string) bool +} + +func Store(conf config.Storage, log *logger.Logger) (Storage, error) { + var st Storage + var err error + switch conf.Provider { + case "s3": + st, err = NewS3Client(conf.S3Endpoint, conf.S3BucketName, conf.S3AccessKeyId, conf.S3SecretAccessKey, log) + case "coordinator": + default: + } + return st, err +} diff --git a/pkg/worker/worker.go b/pkg/worker/worker.go index 6ff68e033..148187287 100644 --- a/pkg/worker/worker.go +++ b/pkg/worker/worker.go @@ -59,7 +59,7 @@ func New(conf config.WorkerConfig, log *logger.Logger) (*Worker, error) { if conf.Worker.Monitoring.IsEnabled() { worker.services[1] = monitoring.New(conf.Worker.Monitoring, h.GetHost(), log) } - st, err := cloud.Store(conf.Storage.Provider, conf.Storage.Key) + st, err := cloud.Store(conf.Storage, log) if err != nil { log.Warn().Err(err).Msgf("cloud storage fail, using no storage") }