diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f2b5672c5..335ee3f5f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -373,7 +373,7 @@ jobs: console.log(release.data.upload_url); return release.data.upload_url - - uses: golemfactory/build-deb-action@main + - uses: golemfactory/build-deb-action@5 id: deb with: debVersion: ${{ steps.version.outputs.version-ext }} diff --git a/Cargo.lock b/Cargo.lock index dcd9a373c6..599776fd2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1163,6 +1163,52 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.1.18" @@ -3452,6 +3498,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime 2.1.0", + "serde", +] + [[package]] name = "hyper" version = "0.14.30" @@ -7476,6 +7532,18 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "test-binary" +version = "3.0.2" +source = "git+https://github.com/golemfactory/test-binary.git#c9ebfa3e257455f8365e042b8838a518f2106169" +dependencies = [ + "camino", + "cargo_metadata 0.15.4", + "once_cell", + "paste", + "thiserror", +] + [[package]] name = "test-case" version = "2.2.2" @@ -8796,7 +8864,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.9.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=b5369b76044f0a1584532f603c542587e15be852#b5369b76044f0a1584532f603c542587e15be852" +source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" dependencies = [ "actix-codec", "awc", @@ -8820,13 +8888,14 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=b5369b76044f0a1584532f603c542587e15be852#b5369b76044f0a1584532f603c542587e15be852" +source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" dependencies = [ "bigdecimal 0.2.2", "chrono", "derive_more", "diesel", "hex", + "humantime-serde", "openssl", "rand 0.8.5", "secp256k1 0.27.0", @@ -9007,12 +9076,14 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.8.26", + "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", "sha3 0.8.2", "shell-words", "signal-hook", "socket2 0.4.10", "structopt", "tempdir", + "test-context", "thiserror", "tokio", "tokio-stream", @@ -9024,10 +9095,12 @@ dependencies = [ "ya-compile-time-utils", "ya-core-model", "ya-counters", + "ya-framework-basic", "ya-gsb-http-proxy", "ya-manifest-utils", + "ya-mock-runtime", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-sb-router", "ya-service-bus", "ya-std-utils", @@ -9072,6 +9145,7 @@ dependencies = [ "async-trait", "awc", "bytes 1.7.1", + "cargo_metadata 0.18.1", "crossterm 0.26.1", "env_logger 0.7.1", "futures 0.3.30", @@ -9089,6 +9163,7 @@ dependencies = [ "sha2 0.8.2", "sha3 0.8.2", "tempdir", + "test-binary", "test-context", "thiserror", "tokio", @@ -9430,6 +9505,36 @@ dependencies = [ "ya-service-bus", ] +[[package]] +name = "ya-mock-runtime" +version = "0.1.0" +dependencies = [ + "actix", + "anyhow", + "async-trait", + "bytes 1.7.1", + "env_logger 0.10.2", + "futures 0.3.30", + "hex", + "log", + "portpicker", + "rand 0.8.5", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "url", + "uuid 0.8.2", + "ya-client-model", + "ya-core-model", + "ya-exe-unit", + "ya-framework-basic", + "ya-runtime-api 0.7.0", + "ya-sb-router", + "ya-service-bus", +] + [[package]] name = "ya-net" version = "0.3.0" @@ -9830,6 +9935,25 @@ dependencies = [ "derive_more", ] +[[package]] +name = "ya-runtime-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0db25811f107d62be6c6ac7444d9c6c3e39714b6f76d72798b66ecce47506f" +dependencies = [ + "anyhow", + "bytes 1.7.1", + "futures 0.3.30", + "log", + "prost 0.10.4", + "prost-build 0.10.4", + "serde", + "serde_json", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "ya-runtime-api" version = "0.7.1" @@ -10096,6 +10220,7 @@ dependencies = [ "test-context", "thiserror", "tokio", + "tokio-stream", "tokio-tar", "tokio-util", "url", @@ -10105,7 +10230,7 @@ dependencies = [ "ya-core-model", "ya-exe-unit", "ya-framework-basic", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-service-bus", "ya-utils-futures", "ya-utils-path", diff --git a/Cargo.toml b/Cargo.toml index a1f39a2e7c..62c16458f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,28 +1,28 @@ [package] -name = "yagna" -version = "0.16.0" -description = "Open platform and marketplace for distributed computations" -readme = "README.md" authors = ["Golem Factory "] +description = "Open platform and marketplace for distributed computations" +edition = "2018" homepage = "https://github.com/golemfactory/yagna/core/serv" -repository = "https://github.com/golemfactory/yagna" license = "GPL-3.0" -edition = "2018" +name = "yagna" +readme = "README.md" +repository = "https://github.com/golemfactory/yagna" +version = "0.16.0" [features] default = ['erc20-driver', 'gftp/bin'] -static-openssl = ["openssl/vendored", "openssl-probe"] dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] +framework-test = ['ya-exe-unit/system-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] +static-openssl = ["openssl/vendored", "openssl-probe"] tos = [] -framework-test = ['ya-exe-unit/framework-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ - "ya-vpn/packet-trace-enable", - "ya-file-logging/packet-trace-enable", - "ya-net/packet-trace-enable", - "ya-service-bus/packet-trace-enable", + "ya-vpn/packet-trace-enable", + "ya-file-logging/packet-trace-enable", + "ya-net/packet-trace-enable", + "ya-service-bus/packet-trace-enable", ] [[bin]] @@ -30,40 +30,40 @@ name = "yagna" path = "core/serv/src/main.rs" [dependencies] +gftp = {workspace = true, optional = true} ya-activity = "0.4" +ya-client-model.workspace = true +ya-client.workspace = true ya-compile-time-utils = "0.2" ya-core-model.workspace = true -ya-dummy-driver = { version = "0.3", optional = true } +ya-dummy-driver = {version = "0.3", optional = true} +ya-erc20-driver = {version = "0.4", optional = true} +ya-fd-metrics = {path = "utils/fd-metrics"} ya-file-logging = "0.1" ya-gsb-api = "0.1" -ya-erc20-driver = { version = "0.4", optional = true } ya-identity = "0.3" ya-market = "0.4" ya-metrics = "0.2" -ya-net = { version = "0.3", features = ["service"] } +ya-net = {version = "0.3", features = ["service"]} ya-payment = "0.3" -ya-persistence = { version = "0.3", features = ["service"] } -ya-sb-proto = { workspace = true } -ya-sb-router = { workspace = true } +ya-persistence = {version = "0.3", features = ["service"]} +ya-sb-proto = {workspace = true} +ya-sb-router = {workspace = true} ya-service-api = "0.1" ya-service-api-derive = "0.2" ya-service-api-interfaces = "0.2" ya-service-api-web = "0.2" -ya-service-bus = { workspace = true } +ya-service-bus = {workspace = true} ya-sgx = "0.2" -ya-utils-path = "0.1" ya-utils-futures.workspace = true -ya-utils-process = { version = "0.2", features = ["lock"] } ya-utils-networking = "0.2" -ya-fd-metrics = { path = "utils/fd-metrics" } +ya-utils-path = "0.1" +ya-utils-process = {version = "0.2", features = ["lock"]} ya-version = "0.2" ya-vpn = "0.2" -ya-client.workspace = true -ya-client-model.workspace = true -gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb -ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests -ya-exe-unit = { version = "0.4", optional = true, path = "exe-unit" } # just to enable conditionally running some tests +ya-exe-unit = {version = "0.4", optional = true, path = "exe-unit"}# just to enable conditionally running some tests +ya-provider = {version = "0.3", optional = true}# just to enable conditionally running some tests actix-rt = "2.7" actix-service = "2" @@ -74,163 +74,161 @@ directories = "2.0.2" dotenv = "0.15.0" futures = "0.3" lazy_static = "1.4" +libsqlite3-sys = {workspace = true} log = "0.4" metrics = "0.12" num_cpus = "1" +openssl-probe = {version = "0.1", optional = true} openssl.workspace = true -openssl-probe = { version = "0.1", optional = true } serde = "1.0" serde_json = "1.0" structopt = "0.3" -tokio = { version = "1", features = ["net"] } -tokio-util = { version = "0.7", features = ["codec"] } -tokio-stream = { version = "0.1.8", features = ["io-util"] } +tokio = {version = "1", features = ["net"]} +tokio-stream = {version = "0.1.8", features = ["io-util"]} +tokio-util = {version = "0.7", features = ["codec"]} url = "2.1.1" -libsqlite3-sys = { workspace = true } - [dev-dependencies] -erc20_processor = { workspace = true } +erc20_processor = {workspace = true} +ya-exe-unit = {version = "0.4", path = "exe-unit"} ya-test-framework = "0.1" -ya-exe-unit = { version = "0.4", path = "exe-unit" } [package.metadata.deb] -name = "golem-requestor" assets = [ - [ - "target/release/yagna", - "usr/bin/", - "755", - ], - [ - "target/release/gftp", - "usr/bin/", - "755", - ], - [ - "README.md", - "usr/share/doc/yagna/", - "644", - ], - [ - "core/serv/README.md", - "usr/share/doc/yagna/service.md", - "644", - ], + [ + "target/release/yagna", + "usr/bin/", + "755", + ], + [ + "target/release/gftp", + "usr/bin/", + "755", + ], + [ + "README.md", + "usr/share/doc/yagna/", + "644", + ], + [ + "core/serv/README.md", + "usr/share/doc/yagna/service.md", + "644", + ], ] conflicts = "ya-provider" -features = ["static-openssl"] -maintainer-scripts = "debian/core" depends = "libgcc1, libc6 (>= 2.23)" extended-description = """The Next Golem Milestone. An open platform and marketplace for distributed computations. """ - +features = ["static-openssl"] +maintainer-scripts = "debian/core" +name = "golem-requestor" [package.metadata.deb.variants.provider] -name = "golem-provider" -replaces = "golem-requestor" -maintainer-scripts = "debian/provider" -features = ["static-openssl"] -depends = "libgcc1, libc6 (>= 2.23)" assets = [ - [ - "target/release/yagna", - "usr/bin/", - "755", - ], - [ - "target/release/ya-provider", - "usr/bin/", - "755", - ], - [ - "target/release/gftp", - "usr/bin/", - "755", - ], - [ - "target/release/exe-unit", - "usr/lib/yagna/plugins/", - "755", - ], - [ - "target/release/golemsp", - "usr/bin/", - "755", - ], - [ - "README.md", - "usr/share/doc/yagna/", - "644", - ], - [ - "core/serv/README.md", - "usr/share/doc/yagna/service.md", - "644", - ], - [ - "agent/provider/readme.md", - "usr/share/doc/yagna/run-provider.md", - "644", - ], + [ + "target/release/yagna", + "usr/bin/", + "755", + ], + [ + "target/release/ya-provider", + "usr/bin/", + "755", + ], + [ + "target/release/gftp", + "usr/bin/", + "755", + ], + [ + "target/release/exe-unit", + "usr/lib/yagna/plugins/", + "755", + ], + [ + "target/release/golemsp", + "usr/bin/", + "755", + ], + [ + "README.md", + "usr/share/doc/yagna/", + "644", + ], + [ + "core/serv/README.md", + "usr/share/doc/yagna/service.md", + "644", + ], + [ + "agent/provider/readme.md", + "usr/share/doc/yagna/run-provider.md", + "644", + ], ] +depends = "libgcc1, libc6 (>= 2.23)" +features = ["static-openssl"] +maintainer-scripts = "debian/provider" +name = "golem-provider" +replaces = "golem-requestor" [workspace.lints.clippy] arc_with_non_send_sync = "allow" -get_first = "allow" blocks_in_conditions = "allow" +get_first = "allow" [workspace] members = [ - "agent/provider", - "core/activity", - "core/gftp", - "core/gsb-api", - "core/identity", - "core/market", - "core/market/resolver", - "core/model", - "core/net", - "core/payment", - "core/payment-driver/base", - "core/payment-driver/dummy", - "core/payment-driver/erc20", - "core/persistence", - "core/serv-api", - "core/serv-api/derive", - "core/serv-api/interfaces", - "core/serv-api/web", - "core/sgx", - "core/version", - "core/vpn", - "exe-unit/components/counters", - "exe-unit/components/gsb-http-proxy", - "exe-unit", - "exe-unit/runtime-api", - "exe-unit/tokio-process-ns", - "exe-unit/components/transfer", - "golem_cli", - "utils/actix_utils", - "utils/agreement-utils", - "utils/cli", - "utils/compile-time-utils", - "utils/file-logging", - "utils/futures", - "utils/manifest-utils", - "utils/manifest-utils/test-utils", - "utils/networking", - "utils/path", - "utils/process", - "utils/std-utils", - "utils/diesel-utils", - "utils/fd-metrics", - "core/metrics", - "test-utils/test-framework", - "test-utils/test-framework/framework-macro", - "test-utils/test-framework/framework-basic", - "test-utils/test-framework/framework-mocks", + "agent/provider", + "core/activity", + "core/gftp", + "core/gsb-api", + "core/identity", + "core/market", + "core/market/resolver", + "core/model", + "core/net", + "core/payment", + "core/payment-driver/base", + "core/payment-driver/dummy", + "core/payment-driver/erc20", + "core/persistence", + "core/serv-api", + "core/serv-api/derive", + "core/serv-api/interfaces", + "core/serv-api/web", + "core/sgx", + "core/version", + "core/vpn", + "exe-unit/components/counters", + "exe-unit/components/gsb-http-proxy", + "exe-unit", + "exe-unit/runtime-api", + "exe-unit/tokio-process-ns", + "exe-unit/components/transfer", + "golem_cli", + "utils/actix_utils", + "utils/agreement-utils", + "utils/cli", + "utils/compile-time-utils", + "utils/file-logging", + "utils/futures", + "utils/manifest-utils", + "utils/manifest-utils/test-utils", + "utils/networking", + "utils/path", + "utils/process", + "utils/std-utils", + "utils/diesel-utils", + "utils/fd-metrics", + "core/metrics", + "test-utils/test-framework", + "test-utils/test-framework/framework-macro", + "test-utils/test-framework/framework-basic", + "test-utils/test-framework/framework-mocks", ] [workspace.dependencies] @@ -238,34 +236,34 @@ members = [ # diesel 1.4.* supports up to 0.23.0, but sqlx 0.5.9 requires 0.22.0 # sqlx 0.5.10 need 0.23.2, so 0.5.9 is last version possible derive_more = "0.99.11" -erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } -erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } +erc20_payment_lib = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} +erc20_processor = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} #erc20_payment_lib = { path = "../../payments/erc20_payment_lib/crates/erc20_payment_lib" } #erc20_processor = { path = "../../payments/erc20_payment_lib" } #erc20_payment_lib = { version = "0.4.7" } #erc20_processor = { version = "0.4.7" } -gftp = { version = "0.4.1", path = "core/gftp" } +gftp = {version = "0.4.1", path = "core/gftp"} hex = "0.4.3" -libsqlite3-sys = { version = "0.26.0", features = ["bundled"] } +libsqlite3-sys = {version = "0.26.0", features = ["bundled"]} openssl = "0.10" rand = "0.8.5" -strum = { version = "0.24", features = ["derive"] } +regex = "1.10.4" +strum = {version = "0.24", features = ["derive"]} trust-dns-resolver = "0.22" url = "2.3.1" -regex = "1.10.4" -ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } -ya-core-model = { version = "0.10", path = "core/model" } -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489" } -ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } -ya-utils-futures = { path = "utils/futures" } +ya-agreement-utils = {version = "0.6", path = "utils/agreement-utils"} +ya-core-model = {version = "0.10", path = "core/model"} +ya-relay-client = {git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489"} +ya-relay-stack = {git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5"} +ya-utils-futures = {path = "utils/futures"} -ya-service-bus = { version = "0.7.3", features = ['tls'] } -ya-sb-router = { version = "0.6.4" } -ya-sb-proto = { version = "0.6.2" } -ya-sb-util = { version = "0.5.1" } -parking_lot = "0.12.3" mime = "0.3.17" +parking_lot = "0.12.3" +ya-sb-proto = {version = "0.6.2"} +ya-sb-router = {version = "0.6.4"} +ya-sb-util = {version = "0.5.1"} +ya-service-bus = {version = "0.7.3", features = ['tls']} # true version is given in patches section ya-client = "0.9" # true version is given in patches section @@ -273,34 +271,34 @@ ya-client-model = "0.7" [patch.crates-io] ## SERVICES -ya-identity = { path = "core/identity" } -ya-net = { path = "core/net" } -ya-market = { path = "core/market" } -ya-market-resolver = { path = "core/market/resolver" } -ya-activity = { path = "core/activity" } -ya-sgx = { path = "core/sgx" } -ya-payment = { path = "core/payment" } -ya-payment-driver = { path = "core/payment-driver/base" } -ya-dummy-driver = { path = "core/payment-driver/dummy" } -ya-erc20-driver = { path = "core/payment-driver/erc20" } -ya-version = { path = "core/version" } -ya-vpn = { path = "core/vpn" } -ya-gsb-api = { path = "core/gsb-api" } +ya-activity = {path = "core/activity"} +ya-dummy-driver = {path = "core/payment-driver/dummy"} +ya-erc20-driver = {path = "core/payment-driver/erc20"} +ya-gsb-api = {path = "core/gsb-api"} +ya-identity = {path = "core/identity"} +ya-market = {path = "core/market"} +ya-market-resolver = {path = "core/market/resolver"} +ya-net = {path = "core/net"} +ya-payment = {path = "core/payment"} +ya-payment-driver = {path = "core/payment-driver/base"} +ya-sgx = {path = "core/sgx"} +ya-version = {path = "core/version"} +ya-vpn = {path = "core/vpn"} ## CORE UTILS -ya-core-model = { path = "core/model" } -ya-persistence = { path = "core/persistence" } -ya-service-api = { path = "core/serv-api" } -ya-service-api-derive = { path = "core/serv-api/derive" } -ya-service-api-interfaces = { path = "core/serv-api/interfaces" } -ya-service-api-web = { path = "core/serv-api/web" } +ya-core-model = {path = "core/model"} +ya-persistence = {path = "core/persistence"} +ya-service-api = {path = "core/serv-api"} +ya-service-api-derive = {path = "core/serv-api/derive"} +ya-service-api-interfaces = {path = "core/serv-api/interfaces"} +ya-service-api-web = {path = "core/serv-api/web"} ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "b5369b76044f0a1584532f603c542587e15be852" } +ya-client = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "b5369b76044f0a1584532f603c542587e15be852" } +ya-client-model = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} #ya-client-model = { path = "../ya-client/model" } -golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415" } +golem-certificate = {git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415"} ## RELAY and networking stack @@ -309,39 +307,38 @@ golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.g #ya-relay-core = { path = "../ya-relay/crates/core" } #ya-relay-proto = { path = "../ya-relay/crates/proto" } - ## OTHERS -gftp = { path = "core/gftp" } -tokio-process-ns = { path = "exe-unit/tokio-process-ns" } -ya-agreement-utils = { path = "utils/agreement-utils" } -ya-std-utils = { path = "utils/std-utils" } -ya-compile-time-utils = { path = "utils/compile-time-utils" } -ya-exe-unit = { path = "exe-unit" } -ya-file-logging = { path = "utils/file-logging" } -ya-manifest-utils = { path = "utils/manifest-utils" } -ya-transfer = { path = "exe-unit/components/transfer" } -ya-utils-actix = { path = "utils/actix_utils" } -ya-utils-cli = { path = "utils/cli" } -ya-utils-networking = { path = "utils/networking" } -ya-utils-path = { path = "utils/path" } -ya-utils-process = { path = "utils/process" } -ya-diesel-utils = { path = "utils/diesel-utils" } -ya-metrics = { path = "core/metrics" } -ya-provider = { path = "agent/provider" } -ya-counters = { path = "exe-unit/components/counters" } -ya-gsb-http-proxy = { path = "exe-unit/components/gsb-http-proxy" } +gftp = {path = "core/gftp"} +tokio-process-ns = {path = "exe-unit/tokio-process-ns"} +ya-agreement-utils = {path = "utils/agreement-utils"} +ya-compile-time-utils = {path = "utils/compile-time-utils"} +ya-counters = {path = "exe-unit/components/counters"} +ya-diesel-utils = {path = "utils/diesel-utils"} +ya-exe-unit = {path = "exe-unit"} +ya-file-logging = {path = "utils/file-logging"} +ya-gsb-http-proxy = {path = "exe-unit/components/gsb-http-proxy"} +ya-manifest-utils = {path = "utils/manifest-utils"} +ya-metrics = {path = "core/metrics"} +ya-provider = {path = "agent/provider"} +ya-std-utils = {path = "utils/std-utils"} +ya-transfer = {path = "exe-unit/components/transfer"} +ya-utils-actix = {path = "utils/actix_utils"} +ya-utils-cli = {path = "utils/cli"} +ya-utils-networking = {path = "utils/networking"} +ya-utils-path = {path = "utils/path"} +ya-utils-process = {path = "utils/process"} ## TEST UTILS -ya-manifest-test-utils = { path = "utils/manifest-utils/test-utils" } -ya-test-framework = { path = "test-utils/test-framework" } -ya-framework-macro = { path = "test-utils/test-framework/framework-macro" } -ya-framework-basic = { path = "test-utils/test-framework/framework-basic" } -ya-framework-mocks = { path = "test-utils/test-framework/framework-mocks" } +ya-framework-basic = {path = "test-utils/test-framework/framework-basic"} +ya-framework-macro = {path = "test-utils/test-framework/framework-macro"} +ya-framework-mocks = {path = "test-utils/test-framework/framework-mocks"} +ya-manifest-test-utils = {path = "utils/manifest-utils/test-utils"} +ya-test-framework = {path = "test-utils/test-framework"} -ethereum-tx-sign = { git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195" } -graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407" } +ethereum-tx-sign = {git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195"} +graphene-sgx = {git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407"} -diesel = { git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563" } +diesel = {git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563"} # Speed up builds on macOS (will be default in next rust version probably) # https://jakedeichert.com/blog/reducing-rust-incremental-compilation-times-on-macos-by-70-percent/ diff --git a/docs/provider/capabilities.md b/docs/provider/capabilities.md index e5ddaf928e..e60d8f00d1 100644 --- a/docs/provider/capabilities.md +++ b/docs/provider/capabilities.md @@ -2,10 +2,19 @@ ## Protocol -| Capability | Yagna package version | Backwards-compatible? | Description | -|-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use (which?) property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | -| Restart Proposal Negotiations | 0.7.0 | Yes | Agent is allowed to restart negotiations, by sending `Counter Proposal`, after he rejected Proposal at some point. Counter-party will receive regular `ProposalEvent` in this case. Only Agent rejecting Proposal has initiative in restarting negotiations, rejected Agent can only wait for this to happen. To indicate, that Proposal rejection isn't final and negotiations can be restarted later, Agent can set `golem.proposal.rejection.is-final` (bool) field in `Reason` structure. If this value is set to false, Agent can free any state related to this negotiation. The same field can be set in `Reason` sent in `Reject Agreement` operation. Requestor can send new counter Proposal after some period of time or propose the same Agreement for the second time. (No change to specification) | -| manifest-support | | | TODO | -| inet | | | TODO | -| start-entrypoint | | | TODO | + +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------| +| Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use `golem.srv.caps.multi-activity` property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | golem.srv.caps.multi-activity | +| Restart Proposal Negotiations | 0.7.0 | Yes | Agent is allowed to restart negotiations, by sending `Counter Proposal`, after he rejected Proposal at some point. Counter-party will receive regular `ProposalEvent` in this case. Only Agent rejecting Proposal has initiative in restarting negotiations, rejected Agent can only wait for this to happen. To indicate, that Proposal rejection isn't final and negotiations can be restarted later, Agent can set `golem.proposal.rejection.is-final` (bool) field in `Reason` structure. If this value is set to false, Agent can free any state related to this negotiation. The same field can be set in `Reason` sent in `Reject Agreement` operation. Requestor can send new counter Proposal after some period of time or propose the same Agreement for the second time. (No change to specification) | | + + +## ExeUnit + + +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------| +| manifest-support | | | TODO | | +| inet | | | TODO | | +| start-entrypoint | | | TODO | | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | `golem.activity.caps.transfer.report-progress` `golem.activity.caps.deploy.report-progress` | diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md new file mode 100644 index 0000000000..6fe412c985 --- /dev/null +++ b/docs/provider/exe-unit/command-progress.md @@ -0,0 +1,49 @@ +# Command progress reporting + +ExeUnit behaves according to specification defined [here](https://golemfactory.github.io/golem-architecture/specs/command-progress.html) +and support progress reporting for commands: `deploy` and `transfer`. + +This document aims to describe implementation details not covered by specification. + +## Specification + + +| Name | Description | +|-----------------------------|----------------------| +| Minimum ExeUnit version | {TODO} | +| Minimum Runtime API version | Always compatible | +| Minimum yagna version | {TODO} | +| Minimum provider version | Always compatible | +| Supported commands | `deploy`, `transfer` | + + +## [ProgressArgs](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#/model-ProgressArgs) + +ExeUnit supports only `update-interval`. If value is not set, `1s` default will be used. + +`update-step` is not implemented. + +## [Runtime event](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#model-RuntimeEventKindProgress) + +### Steps + +`Deploy` and `transfer` command consist of only single step. + +### Progress + +- Progress is reported as `Bytes`. Fields is never a `None`. +- Size of file is always checked and put as second element of tuple. +- Initially `Size` element of tuple is set to `None` and if progress with `message` field is sent + than it can be received by Requestor agent + +### Message + +Two messages are currently possible: +- `Deployed image from cache` +- `Retry in {}s because of error: {err}` - indicates error during transfer, which will result in retry. + +When sending message, the rest of `CommandProgress` structure fields will be set to latest values. + +## Requestor Example + +PoC implementation using yapapi: https://github.com/golemfactory/yapapi/pull/1153 diff --git a/docs/readme.md b/docs/readme.md index 3de0b8ae24..7407a538c0 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,4 +1,23 @@ # Developer documentation -[Provider](./provider/architecture.md) -[Payment Driver](./../core/payment-driver/erc20/Readme.md) \ No newline at end of file +- [Architecture](https://golemfactory.github.io/golem-architecture/) +- [REST API specification](https://golemfactory.github.io/ya-client/) +- Developer guides + - [Installation](./provider/overview.md#installation) + - [Logging guidelines](./logging-guidelines.md) +- Implementation documentation + - Overview + - [Provider](./provider/architecture.md) + - [ExeUnit](./provider/exe-unit/exe-units.md) + - Yagna + - Identity + - Market + - Activity + - Payments + - [Payment Driver](./../core/payment-driver/erc20/Readme.md) + - Net + - GSB + - VPN + - Runtimes + - ya-runtime-vm + - ya-runtime-wasm \ No newline at end of file diff --git a/docs/yagna/capabilities.md b/docs/yagna/capabilities.md index 2844de7103..fd867bb1ec 100644 --- a/docs/yagna/capabilities.md +++ b/docs/yagna/capabilities.md @@ -6,6 +6,7 @@ Capabilities requiring Provider agent support are listed [here](../provider/capa ## Yagna API -| Capability | Yagna package version | Backwards-compatible? | Description | -|-------------|-----------------------|-----------------------|----------------------------------------------------------------------------| -| Cors Policy | 0.12.0 | Yes | Yagna is able to respond with Cors headers. [Spec](./capabilities/cors.md) | +| Capability | Yagna package version | Backwards-compatible? | Description | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------| +| Cors Policy | 0.12.0 | Yes | Yagna is able to respond with Cors headers. [Spec](./capabilities/cors.md) | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | \ No newline at end of file diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index b47daadeae..ff1eb42248 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "ya-exe-unit" -version = "0.4.0" authors = ["Golem Factory "] edition = "2018" +name = "ya-exe-unit" +version = "0.4.0" [lib] name = "ya_exe_unit" @@ -13,64 +13,64 @@ name = "exe-unit" path = "src/bin.rs" [features] -default = ['compat-deployment'] compat-deployment = [] +default = ['compat-deployment'] +packet-trace-enable = ["ya-packet-trace/enable"] sgx = [ - 'graphene-sgx', - 'openssl/vendored', - 'secp256k1/rand', - 'ya-client-model/sgx', - 'ya-core-model/sgx', - 'ya-transfer/sgx', + 'graphene-sgx', + 'openssl/vendored', + 'secp256k1/rand', + 'ya-client-model/sgx', + 'ya-core-model/sgx', + 'ya-transfer/sgx', ] -packet-trace-enable = ["ya-packet-trace/enable"] -framework-test = ["ya-transfer/framework-test"] +system-test = ["ya-transfer/system-test"] [dependencies] -ya-agreement-utils = { workspace = true } -ya-manifest-utils = { version = "0.2" } +ya-agreement-utils = {workspace = true} ya-client-model.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { workspace = true, features = ["activity", "appkey"] } -ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ - "server", -] } -ya-service-bus = { workspace = true } +ya-core-model = {workspace = true, features = ["activity", "appkey"]} +ya-counters = {path = "../exe-unit/components/counters", features = ["os"]} +ya-gsb-http-proxy = {path = "../exe-unit/components/gsb-http-proxy"} +ya-manifest-utils = {version = "0.2"} +ya-packet-trace = {git = "https://github.com/golemfactory/ya-packet-trace"} +ya-runtime-api = {version = "0.7", path = "runtime-api", features = [ + "server", +]} +ya-service-bus = {workspace = true} +ya-std-utils = "0.1" ya-transfer = "0.3" -ya-utils-path = "0.1" ya-utils-futures.workspace = true -ya-std-utils = "0.1" -ya-utils-networking = { version = "0.2", default-features = false, features = [ - "dns", - "vpn", -] } +ya-utils-networking = {version = "0.2", default-features = false, features = [ + "dns", + "vpn", +]} +ya-utils-path = "0.1" ya-utils-process = "0.3" -ya-packet-trace = { git = "https://github.com/golemfactory/ya-packet-trace" } -ya-gsb-http-proxy = { path = "../exe-unit/components/gsb-http-proxy" } -ya-counters = { path = "../exe-unit/components/counters", features = ["os"] } -actix = { version = "0.13", default-features = false } +actix = {version = "0.13", default-features = false} actix-rt = "2.7" anyhow = "1.0" async-trait = "0.1.24" bytes = "1" chrono = "0.4" derivative = "2.1" -derive_more = { workspace = true } +derive_more = {workspace = true} dotenv = "0.15.0" -flexi_logger = { version = "0.22", features = ["colors"] } +flexi_logger = {version = "0.22", features = ["colors"]} futures = "0.3" -graphene-sgx = { version = "0.3.3", optional = true } +graphene-sgx = {version = "0.3.3", optional = true} hex = "0.4.2" ipnet = "2.3" lazy_static = "1.4.0" log = "0.4" -openssl = { workspace = true, optional = true } +openssl = {workspace = true, optional = true} rand = "0.8.5" regex = "1.5" -reqwest = { version = "0.11", optional = false, features = ["stream"] } -secp256k1 = { version = "0.27.0", optional = true } -serde = { version = "^1.0", features = ["derive"] } +reqwest = {version = "0.11", optional = false, features = ["stream"]} +secp256k1 = {version = "0.27.0", optional = true} +serde = {version = "^1.0", features = ["derive"]} serde_json = "1.0" serde_yaml = "0.8" sha3 = "0.8.2" @@ -79,34 +79,39 @@ socket2 = "0.4" structopt = "0.3" thiserror = "1.0" # keep the "rt-multi-thread" feature -tokio = { version = "1", features = [ - "process", - "signal", - "time", - "net", - "rt-multi-thread", -] } -tokio-util = { version = "0.7.2", features = ["codec", "net"] } -tokio-stream = { version = "0.1.8", features = ["io-util", "sync"] } +async-stream = "0.3.5" +tokio = {version = "1", features = [ + "process", + "signal", + "time", + "net", + "rt-multi-thread", +]} +tokio-stream = {version = "0.1.8", features = ["io-util", "sync"]} +tokio-util = {version = "0.7.2", features = ["codec", "net"]} +trust-dns-resolver = {workspace = true} url = "2.1" yansi = "0.5.0" -trust-dns-resolver = { workspace = true } -async-stream = "0.3.5" [dev-dependencies] -ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ - "codec", - "server", -] } -ya-sb-router = { workspace = true } +ya-runtime-api = {version = "0.7", path = "runtime-api", features = [ + "codec", + "server", +]} +ya-sb-router = {workspace = true} actix-files = "0.6" actix-web = "4" env_logger = "0.7" rustyline = "7.0.0" +serial_test = {git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} sha3 = "0.8.2" shell-words = "1.0.0" tempdir = "0.3.7" +test-context = "0.1.4" + +ya-framework-basic = {version = "0.1"} +ya-mock-runtime = {path = "components/mock-runtime"} [lints] workspace = true diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml new file mode 100644 index 0000000000..564a6e5e67 --- /dev/null +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -0,0 +1,43 @@ +[package] +authors = ["Golem Factory "] +description = "Mock runtime for testing purposes and set of libraries for testing ExeUnits in tests." +edition = "2021" +name = "ya-mock-runtime" +version = "0.1.0" + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "ya-mock-runtime" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.31" +bytes = "1.0" +env_logger = "0.10" +futures = {version = "0.3"} +log = "0.4" +serde = {version = "1.0", features = ["derive"]} +serde_json = "1.0" +thiserror = "1.0" +tokio = {version = "1", features = ["io-std", "rt", "process", "sync", "macros", "time"]} +tokio-util = {version = "0.7", features = ["codec"]} +url = "2.3" + +ya-runtime-api = "0.7" + +# Dependancies for ExeUnit testing utils +ya-client-model.workspace = true +ya-core-model = {workspace = true, features = ["activity", "appkey"]} +ya-exe-unit = "0.4" +ya-framework-basic = {version = "0.1"} +ya-sb-router = "0.6" +ya-service-bus = {workspace = true} + +actix = {version = "0.13", default-features = false} +async-trait = "0.1.77" +hex = "0.4.3" +portpicker = "0.1.1" +rand = "0.8.5" +uuid = {version = "0.8.2", features = ["v4"]} diff --git a/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json b/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json new file mode 100644 index 0000000000..93f43f2d54 --- /dev/null +++ b/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json @@ -0,0 +1,9 @@ +[ + { + "name": "ya-mock-runtime", + "version": "0.1.0", + "supervisor-path": "exe-unit", + "runtime-path": "ya-mock-runtime", + "description": "Mock runtime for testing purposes" + } +] \ No newline at end of file diff --git a/exe-unit/runtime-api/examples/runtime-server-mock.rs b/exe-unit/components/mock-runtime/src/lib.rs similarity index 56% rename from exe-unit/runtime-api/examples/runtime-server-mock.rs rename to exe-unit/components/mock-runtime/src/lib.rs index 496b741a17..ee9ed1bab4 100644 --- a/exe-unit/runtime-api/examples/runtime-server-mock.rs +++ b/exe-unit/components/mock-runtime/src/lib.rs @@ -1,20 +1,19 @@ +pub mod testing; + use futures::future::BoxFuture; use futures::prelude::*; use futures::FutureExt; use std::clone::Clone; -use std::env; use std::sync::{Arc, Mutex}; use std::time::Duration; use ya_runtime_api::server::*; -// server - -struct RuntimeMock +pub struct RuntimeMock where H: RuntimeHandler, { - handler: H, + pub handler: H, } impl RuntimeService for RuntimeMock { @@ -61,14 +60,15 @@ impl RuntimeService for RuntimeMock { // client // holds last received status -struct EventMock(Arc>); +#[derive(Clone, Default)] +pub struct EventMock(Arc>); impl EventMock { - fn new() -> Self { - Self(Arc::new(Mutex::new(Default::default()))) + pub fn new() -> Self { + Self::default() } - fn get_last_status(&self) -> ProcessStatus { + pub fn get_last_status(&self) -> ProcessStatus { self.0.lock().unwrap().clone() } } @@ -84,46 +84,3 @@ impl RuntimeHandler for EventMock { future::ready(()).boxed() } } - -impl Clone for EventMock { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - if env::var("RUST_LOG").is_err() { - env::set_var("RUST_LOG", "debug") - } - env_logger::init(); - if env::var("X_SERVER").is_ok() { - run(|event_emitter| RuntimeMock { - handler: event_emitter, - }) - .await - } else { - use tokio::process::Command; - let exe = env::current_exe().unwrap(); - - let mut cmd = Command::new(exe); - cmd.env("X_SERVER", "1"); - let events = EventMock::new(); - let c = spawn(cmd, events.clone()).await?; - log::debug!("hello_result={:?}", c.hello("0.0.0x").await); - let run = RunProcess { - bin: "sleep".to_owned(), - args: vec!["10".to_owned()], - ..Default::default() - }; - let sleep_1 = c.run_process(run.clone()); - let sleep_2 = c.run_process(run.clone()); - let sleep_3 = c.run_process(run); - log::info!("start sleep1"); - log::info!("sleep1={:?}", sleep_1.await); - log::info!("start sleep2 sleep3"); - log::info!("sleep23={:?}", future::join(sleep_2, sleep_3).await); - log::info!("last status: {:?}", events.get_last_status()); - } - Ok(()) -} diff --git a/exe-unit/components/mock-runtime/src/main.rs b/exe-unit/components/mock-runtime/src/main.rs new file mode 100644 index 0000000000..509263c203 --- /dev/null +++ b/exe-unit/components/mock-runtime/src/main.rs @@ -0,0 +1,42 @@ +use futures::future; +use std::env; + +use ya_mock_runtime::{EventMock, RuntimeMock}; +use ya_runtime_api::server::{run, spawn, RunProcess, RuntimeService}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "debug") + } + env_logger::init(); + if env::var("X_SERVER").is_ok() { + run(|event_emitter| RuntimeMock { + handler: event_emitter, + }) + .await + } else { + use tokio::process::Command; + let exe = env::current_exe().unwrap(); + + let mut cmd = Command::new(exe); + cmd.env("X_SERVER", "1"); + let events = EventMock::new(); + let c = spawn(cmd, events.clone()).await?; + log::debug!("hello_result={:?}", c.hello("0.0.0x").await); + let run = RunProcess { + bin: "sleep".to_owned(), + args: vec!["10".to_owned()], + ..Default::default() + }; + let sleep_1 = c.run_process(run.clone()); + let sleep_2 = c.run_process(run.clone()); + let sleep_3 = c.run_process(run); + log::info!("start sleep1"); + log::info!("sleep1={:?}", sleep_1.await); + log::info!("start sleep2 sleep3"); + log::info!("sleep23={:?}", future::join(sleep_2, sleep_3).await); + log::info!("last status: {:?}", events.get_last_status()); + } + Ok(()) +} diff --git a/exe-unit/components/mock-runtime/src/testing.rs b/exe-unit/components/mock-runtime/src/testing.rs new file mode 100644 index 0000000000..7b2c833428 --- /dev/null +++ b/exe-unit/components/mock-runtime/src/testing.rs @@ -0,0 +1,3 @@ +mod exe_unit_ext; + +pub use exe_unit_ext::{create_exe_unit, exe_unit_config, ExeUnitExt, ExeUnitHandle}; diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs new file mode 100644 index 0000000000..8559a20c27 --- /dev/null +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -0,0 +1,238 @@ +use actix::Addr; +use anyhow::{anyhow, bail}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::broadcast; +use url::Url; +use uuid::Uuid; + +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::{ExeScriptCommand, State, StatePair}; +use ya_core_model::activity; +use ya_exe_unit::message::{GetBatchResults, GetState, GetStateResponse, Shutdown, ShutdownReason}; +use ya_exe_unit::runtime::process::RuntimeProcess; +use ya_exe_unit::{exe_unit, ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; +use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; +use ya_service_bus::RpcEnvelope; + +#[async_trait::async_trait] +pub trait ExeUnitExt { + async fn exec( + &self, + batch_id: Option, + exe_script: Vec, + ) -> anyhow::Result; + + async fn deploy(&self, progress: Option) -> anyhow::Result; + async fn start(&self, args: Vec) -> anyhow::Result; + + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()>; + + /// Waits until ExeUnit will be ready to receive commands. + async fn await_init(&self) -> anyhow::Result<()>; +} + +#[derive(Debug, Clone)] +pub struct ExeUnitHandle { + pub addr: Addr>, + pub config: Arc, +} + +impl ExeUnitHandle { + pub fn new( + addr: Addr>, + config: ExeUnitConfig, + ) -> anyhow::Result { + Ok(ExeUnitHandle { + addr, + config: Arc::new(config), + }) + } + + pub async fn finish_notifier(&self) -> anyhow::Result> { + Ok(self.addr.send(FinishNotifier {}).await??) + } + + pub async fn shutdown(&self) -> anyhow::Result<()> { + let finish = self.finish_notifier().await; + + log::info!("Waiting for shutdown.."); + + self.addr + .send(Shutdown(ShutdownReason::Finished)) + .await + .ok(); + if let Ok(mut finish) = finish { + finish.recv().await?; + } + Ok(()) + } +} + +#[async_trait::async_trait] +impl AsyncDroppable for ExeUnitHandle { + async fn async_drop(&self) { + self.shutdown().await.ok(); + } +} + +pub fn exe_unit_config( + temp_dir: &Path, + agreement_path: &Path, + binary: impl AsRef, +) -> ExeUnitConfig { + ExeUnitConfig { + args: RunArgs { + agreement: agreement_path.to_path_buf(), + cache_dir: temp_dir.join("cache"), + work_dir: temp_dir.join("work"), + }, + binary: binary.as_ref().to_path_buf(), + runtime_args: vec![], + supervise: SuperviseCli { + hardware: false, + image: false, + }, + sec_key: None, + requestor_pub_key: None, + service_id: Some(Uuid::new_v4().to_simple().to_string()), + report_url: None, + } +} + +pub async fn create_exe_unit( + config: ExeUnitConfig, + ctx: &mut DroppableTestContext, +) -> anyhow::Result { + // If activity id was provided, ExeUnit will bind endpoints on remote GSB. + // For this to work we need to setup gsb router. + if config.service_id.is_some() { + let gsb_url = match std::env::consts::FAMILY { + "unix" => Url::from_str(&format!( + "unix://{}/gsb.sock", + config.args.work_dir.display() + ))?, + _ => Url::from_str(&format!( + "tcp://127.0.0.1:{}", + portpicker::pick_unused_port().ok_or(anyhow!("No ports free"))? + ))?, + }; + + if gsb_url.scheme() == "unix" { + let dir = PathBuf::from_str(gsb_url.path())? + .parent() + .map(|path| path.to_path_buf()) + .ok_or(anyhow!("`gsb_url` unix socket has no parent directory."))?; + fs::create_dir_all(dir)?; + } + + // GSB takes url from this variable and we can't set it directly. + std::env::set_var("GSB_URL", gsb_url.to_string()); + ya_sb_router::bind_gsb_router(Some(gsb_url.clone())) + .await + .map_err(|e| anyhow!("Error binding service bus router to '{}': {e}", &gsb_url))?; + } + + let exe = exe_unit(config.clone()).await.unwrap(); + let handle = ExeUnitHandle::new(exe, config)?; + ctx.register(handle.clone()); + Ok(handle) +} + +#[async_trait::async_trait] +impl ExeUnitExt for ExeUnitHandle { + async fn exec( + &self, + batch_id: Option, + exe_script: Vec, + ) -> anyhow::Result { + log::debug!("Executing commands: {:?}", exe_script); + + let batch_id = if let Some(batch_id) = batch_id { + batch_id + } else { + hex::encode(rand::random::<[u8; 16]>()) + }; + + let msg = activity::Exec { + activity_id: self.config.service_id.clone().unwrap_or_default(), + batch_id: batch_id.clone(), + exe_script, + timeout: None, + }; + self.addr + .send(RpcEnvelope::with_caller(String::new(), msg)) + .await + .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))? + .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))?; + Ok(batch_id) + } + + async fn deploy(&self, progress: Option) -> anyhow::Result { + Ok(self + .exec( + None, + vec![ExeScriptCommand::Deploy { + net: vec![], + progress, + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec![], + }], + ) + .await + .unwrap()) + } + + async fn start(&self, args: Vec) -> anyhow::Result { + Ok(self + .exec(None, vec![ExeScriptCommand::Start { args }]) + .await + .unwrap()) + } + + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()> { + let delay = Duration::from_secs_f32(0.5); + loop { + match self + .addr + .send(GetBatchResults { + batch_id: batch_id.to_string(), + idx: None, + }) + .await + { + Ok(results) => { + if let Some(last) = results.0.last() { + if last.is_batch_finished { + return Ok(()); + } + } + } + Err(e) => bail!("Waiting for batch: {batch_id}. Error: {e}"), + } + tokio::time::sleep(delay).await; + } + } + + async fn await_init(&self) -> anyhow::Result<()> { + let delay = Duration::from_secs_f32(0.3); + loop { + match self.addr.send(GetState).await { + Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, + Ok(GetStateResponse(StatePair(State::Terminated, _))) + | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) + | Err(_) => { + log::error!("ExeUnit has terminated"); + bail!("ExeUnit has terminated"); + } + _ => tokio::time::sleep(delay).await, + } + } + Ok(()) + } +} diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 1c09e3156b..ad4f965ef5 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] ya-client = { workspace = true } ya-client-model = { workspace = true } -ya-core-model = { workspace = true, features = ["gftp"] } +ya-core-model = { workspace = true, features = ["activity", "gftp"] } ya-service-bus = { workspace = true } ya-utils-path = { version = "0.1", path = "../../../utils/path" } ya-utils-futures.workspace = true @@ -43,6 +43,7 @@ tokio-util = { version = "0.7", features = ["io"] } url = "2.1.1" walkdir = "2.3.1" async-trait = "0.1.74" +tokio-stream = { version = "0.1.14", features = ["sync"] } [target.'cfg(target_family = "unix")'.dependencies] awc = { version = "3", features = ["openssl"] } @@ -59,7 +60,7 @@ sgx = [ 'ya-core-model/sgx', 'reqwest/trust-dns', ] -framework-test = [] +system-test = [] [dependencies.zip] version = "0.5.6" diff --git a/exe-unit/components/transfer/src/http.rs b/exe-unit/components/transfer/src/http.rs index 97bb1b2026..03e1acfd7e 100644 --- a/exe-unit/components/transfer/src/http.rs +++ b/exe-unit/components/transfer/src/http.rs @@ -119,10 +119,6 @@ impl TransferProvider for HttpTransferProvider { url: &Url, ctx: &TransferContext, ) -> LocalBoxFuture<'a, Result<(), Error>> { - if ctx.state.offset() == 0 { - return futures::future::ok(()).boxed_local(); - } - let url = url.clone(); let state = ctx.state.clone(); @@ -137,8 +133,13 @@ impl TransferProvider for HttpTransferProvider { .get(header::CONTENT_LENGTH) .and_then(|v| v.to_str().ok().and_then(|s| u64::from_str(s).ok())); + match &size { + None => log::info!("File size unknown. Http source server didn't respond with CONTENT_LENGTH header."), + Some(size) => log::info!("Http source size reported by server: {size} B"), + }; + state.set_size(size); - if !ranges { + if state.offset() != 0 && !ranges { log::warn!("Transfer resuming is not supported by the server"); state.set_offset(0); } diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index 34e055680a..4cca1ff186 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -25,19 +25,19 @@ use futures::prelude::*; use futures::task::{Context, Poll}; use url::Url; -use crate::error::Error; - pub use crate::archive::{archive, extract, ArchiveFormat}; pub use crate::container::ContainerTransferProvider; +use crate::error::Error; pub use crate::file::{DirTransferProvider, FileTransferProvider}; pub use crate::gftp::GftpTransferProvider; +use crate::hash::with_hash_stream; pub use crate::http::HttpTransferProvider; pub use crate::location::{TransferUrl, UrlExt}; +use crate::progress::{progress_report_channel, ProgressReporter}; pub use crate::progress::{wrap_sink_with_progress_reporting, wrap_stream_with_progress_reporting}; pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; -use crate::hash::with_hash_stream; use ya_client_model::activity::TransferArgs; /// Transfers data from `stream` to a `TransferSink` @@ -73,17 +73,22 @@ where log::debug!("Transferring from offset: {}", ctx.state.offset()); let stream = with_hash_stream(src.source(&src_url.url, ctx), src_url, dst_url, ctx)?; - let sink = dst.destination(&dst_url.url, ctx); + let sink = progress_report_channel(dst.destination(&dst_url.url, ctx), ctx); transfer(stream, sink).await?; Ok::<_, Error>(()) }; match fut.await { - Ok(val) => return Ok(val), + Ok(val) => { + return Ok(val); + } Err(err) => match ctx.state.delay(&err) { Some(delay) => { - log::warn!("Retrying in {}s because: {}", delay.as_secs_f32(), err); + let msg = format!("Retry in {}s because of error: {err}", delay.as_secs_f32()); + log::warn!("{}", msg); + + ctx.progress.report_message(msg); tokio::time::sleep(delay).await; } None => return Err(err), @@ -296,6 +301,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, + pub progress: ProgressReporter, } impl TransferContext { @@ -304,7 +310,15 @@ impl TransferContext { let state = TransferState::default(); state.set_offset(offset); - Self { args, state } + Self { + args, + state, + progress: ProgressReporter::default(), + } + } + + pub fn reporter(&self) -> ProgressReporter { + self.progress.clone() } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 71edaa7419..03f1d6fd04 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -1,11 +1,112 @@ use crate::error::Error; use crate::{abortable_sink, abortable_stream, TransferSink, TransferStream}; use crate::{TransferContext, TransferData}; + use futures::{SinkExt, StreamExt, TryFutureExt}; +use std::sync::Arc; +use std::time::Duration; use tokio::task::spawn_local; +use tokio::time::Instant; + +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::CommandProgress; type Stream = TransferStream; +#[derive(Debug, Clone)] +pub struct ProgressConfig { + /// Channel for watching for transfer progress. + pub progress: tokio::sync::broadcast::Sender, + pub progress_args: ProgressArgs, +} + +#[derive(Default, Clone)] +pub struct ProgressReporter { + config: ProgressArgs, + inner: Arc>>, +} + +struct ProgressImpl { + pub report: tokio::sync::broadcast::Sender, + pub last: CommandProgress, + pub last_send: Instant, +} + +impl ProgressReporter { + pub fn next_step(&self) { + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last.step.0 += 1; + inner.last.progress = (0, None); + inner.last_send = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok() + }); + } + + /// TODO: implement `update_step` + pub fn report_progress(&self, progress: u64, size: Option) { + let update_interval: Duration = self + .config + .update_interval + .map(Into::into) + .unwrap_or(Duration::from_secs(1)); + let _update_step = self.config.update_step; + + if let Some(inner) = self.inner.lock().unwrap().as_mut() { + inner.last.progress = (progress, size); + if inner.last_send + update_interval <= Instant::now() { + inner.last_send = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok(); + } + } + } + + pub fn report_message(&self, message: String) { + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last_send = Instant::now(); + inner + .report + .send(CommandProgress { + message: Some(message), + ..inner.last.clone() + }) + .ok() + }); + } + + pub fn register_reporter( + &mut self, + args: Option, + steps: usize, + unit: Option, + ) { + if let Some(args) = args { + self.config = args.progress_args; + *(self.inner.lock().unwrap()) = Some(ProgressImpl { + report: args.progress, + last: CommandProgress { + step: (0, steps), + message: None, + progress: (0, None), + unit, + }, + last_send: Instant::now(), + }); + } + } +} + /// Wraps a stream to report progress. /// The `report` function is called with the current offset and the total size. /// The total size is 0 if the size is unknown. (For example, when the source is a directory.) @@ -48,6 +149,13 @@ where type Sink = TransferSink; +pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { + let report = ctx.reporter(); + wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { + report.report_progress(progress, size) + }) +} + /// Wraps a sink to report progress. /// The `report` function is called with the current offset and the total size. /// The total size is 0 if the size is unknown. (For example, when the source is a directory.) @@ -91,3 +199,98 @@ where sink } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Instant; + + use tokio::time::Duration; + + #[actix_rt::test] + async fn test_progress_reporter_interval() { + // Note: This test is time dependent and you can expect it to fail on very slow machines. + // If this happens, you could scale intervals by increasing `loop_interval`. Rather you + // shouldn't touch relations between these variables, if you don't know what are you doing. + let loop_interval = 200u64; + let update_interval = 10 * loop_interval; + let offset = loop_interval / 2; + let margin = loop_interval * 8 / 10; + + let mut report = ProgressReporter::default(); + let (tx, mut rx) = tokio::sync::broadcast::channel(10); + report.register_reporter( + Some(ProgressConfig { + progress: tx, + progress_args: ProgressArgs { + update_interval: Some(Duration::from_millis(update_interval)), + update_step: None, + }, + }), + 2, + Some("Bytes".to_string()), + ); + + let size = 200; + let mut before = Instant::now(); + tokio::task::spawn_local(async move { + for step in 0..2 { + tokio::time::sleep(Duration::from_millis(offset)).await; + + for i in 0..=size { + report.report_progress(i, Some(size)); + tokio::time::sleep(Duration::from_millis(loop_interval)).await; + } + if step == 0 { + report.next_step(); + } + } + report.report_message("Finished".to_string()); + }); + + let mut counter = 0; + let mut step = 0; + while let Ok(event) = rx.recv().await { + //println!("{event:?}"); + + counter += 1; + let update = Instant::now().duration_since(before); + before = Instant::now(); + let diff = if update > Duration::from_millis(update_interval + offset) { + update - Duration::from_millis(update_interval + offset) + } else { + Duration::from_millis(update_interval + offset) - update + }; + + assert!(diff <= Duration::from_millis(margin)); + + // `ProgressReporter` should ignore 10 messages in each loop. + assert_eq!(event.progress.0, counter * 10); + assert_eq!(event.progress.1, Some(size)); + assert_eq!(event.step, (step, 2)); + assert_eq!(event.unit, Some("Bytes".to_string())); + assert_eq!(event.message, None); + + if counter == 20 { + if step == 1 { + break; + } + + counter = 0; + step += 1; + + // Skip step change event + rx.recv().await.unwrap(); + before = Instant::now(); + } + } + + // Reporting message will result in event containing progress adn step from previous event. + let last = rx.recv().await.unwrap(); + //println!("{last:?}"); + assert_eq!(last.message, Some("Finished".to_string())); + assert_eq!(last.progress.0, size); + assert_eq!(last.progress.1, Some(size)); + assert_eq!(last.step, (1, 2)); + } +} diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index d3eb77e7f4..8d4bd9503c 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -5,16 +5,20 @@ use std::rc::Rc; use actix::prelude::*; use futures::future::Abortable; +use futures::{Sink, StreamExt, TryStreamExt}; use url::Url; use crate::cache::{Cache, CachePath}; use crate::error::Error; use crate::error::Error as TransferError; +pub use crate::progress::ProgressConfig; use crate::{ transfer_with, ContainerTransferProvider, FileTransferProvider, GftpTransferProvider, HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; +use ya_client_model::activity::exe_script_command::ProgressArgs; +pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; use ya_runtime_api::deploy::ContainerVolume; use ya_utils_futures::abort::Abort; @@ -35,12 +39,14 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug, Message)] +#[derive(Debug, Message, Default)] #[rtype(result = "Result<()>")] pub struct TransferResource { pub from: String, pub to: String, pub args: TransferArgs, + /// Progress reporting configuration. `None` means that there will be no progress updates. + pub progress_config: Option, } #[derive(Message)] @@ -53,10 +59,92 @@ impl AddVolumes { } } -#[derive(Clone, Debug, Message)] +#[derive(Debug, Message, Default)] #[rtype(result = "Result>")] pub struct DeployImage { pub task_package: Option, + /// Progress reporting configuration. `None` means that there will be no progress updates. + pub progress_config: Option, +} + +impl DeployImage { + pub fn with_package(task_package: &str) -> DeployImage { + DeployImage { + task_package: Some(task_package.to_string()), + progress_config: None, + } + } +} + +pub trait ForwardProgressToSink { + fn progress_config_mut(&mut self) -> &mut Option; + + fn forward_progress( + &mut self, + args: &ProgressArgs, + sender: impl Sink + 'static, + ) { + let progress_args = self.progress_config_mut(); + let rx = match progress_args { + None => { + let (tx, rx) = tokio::sync::broadcast::channel(50); + *progress_args = Some(ProgressConfig { + progress: tx, + progress_args: args.clone(), + }); + rx + } + Some(args) => args.progress.subscribe(), + }; + + tokio::task::spawn_local(async move { + tokio_stream::wrappers::BroadcastStream::new(rx) + .map_err(|e| Error::Other(e.to_string())) + .forward(sender) + .await + .ok() + }); + } +} + +impl ForwardProgressToSink for DeployImage { + fn progress_config_mut(&mut self) -> &mut Option { + &mut self.progress_config + } +} + +impl ForwardProgressToSink for TransferResource { + fn progress_config_mut(&mut self) -> &mut Option { + &mut self.progress_config + } +} + +impl DeployImage { + pub fn forward_progress( + &mut self, + args: &ProgressArgs, + sender: impl Sink + 'static, + ) { + let rx = match &self.progress_config { + None => { + let (tx, rx) = tokio::sync::broadcast::channel(50); + self.progress_config = Some(ProgressConfig { + progress: tx, + progress_args: args.clone(), + }); + rx + } + Some(args) => args.progress.subscribe(), + }; + + tokio::task::spawn_local(async move { + tokio_stream::wrappers::BroadcastStream::new(rx) + .map_err(|e| Error::Other(e.to_string())) + .forward(sender) + .await + .ok() + }); + } } #[derive(Clone, Debug, Message)] @@ -160,6 +248,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, + _ctx: TransferContext, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) @@ -181,6 +270,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, + ctx: TransferContext, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); @@ -191,9 +281,6 @@ impl TransferService { hash: None, }; - let ctx = TransferContext::default(); - ctx.state.retry_with(self.deploy_retry.clone()); - // Using partially downloaded image from previous executions could speed up deploy // process, but it comes with the cost: If image under URL changed, Requestor will get // error on the end. This can result with Provider being perceived as unreliable. @@ -211,6 +298,8 @@ impl TransferService { let fut = async move { if path.exists() { log::info!("Deploying cached image: {:?}", path); + ctx.reporter() + .report_message("Deployed image from cache".to_string()); return Ok(Some(path)); } @@ -269,11 +358,16 @@ impl Handler for TransferService { log::info!("Deploying from {:?} to {:?}", src_url.url, path); + let mut ctx = TransferContext::default(); + ctx.state.retry_with(self.deploy_retry.clone()); + ctx.progress + .register_reporter(deploy.progress_config, 1, Some("Bytes".to_string())); + #[cfg(not(feature = "sgx"))] - return self.deploy_no_sgx(src_url, src_name, path); + return self.deploy_no_sgx(src_url, src_name, path, ctx); #[cfg(feature = "sgx")] - return self.deploy_sgx(src_url, src_name, path); + return self.deploy_sgx(src_url, src_name, path, ctx); } } @@ -286,8 +380,10 @@ impl Handler for TransferService { let src = actor_try!(self.provider(&src_url)); let dst = actor_try!(self.provider(&dst_url)); - let ctx = TransferContext::default(); + let mut ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); + ctx.progress + .register_reporter(msg.progress_config, 1, Some("Bytes".to_string())); let (abort, reg) = Abort::new_pair(); diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 269d9bbc7d..229fabdb75 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -1,11 +1,16 @@ use actix::Actor; +use futures::channel::mpsc; +use futures::SinkExt; use std::env; use std::time::Duration; use test_context::test_context; use tokio::time::sleep; +use tokio_stream::StreamExt; +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::CommandProgress; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; @@ -13,11 +18,11 @@ use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, Transf /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("deploy-restart")?; let temp_dir = dir.path(); @@ -31,10 +36,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re std::fs::create_dir_all(dir)?; } - let chunk_size = 4096_usize; - let chunk_count = 1024 * 10; - - let hash = generate_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 1024 * 10); log::debug!("Starting HTTP servers"); let path = temp_dir.to_path_buf(); @@ -42,10 +44,10 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -64,20 +66,87 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re }); log::info!("[>>] Deployment with hash verification"); - let result = addr - .send(DeployImage { - task_package: task_package.clone(), - }) - .await?; + let result = addr.send(DeployImage::with_package(&task_package)).await?; log::info!("Deployment stopped"); assert!(result.is_err()); log::info!("Re-deploying the same image"); - addr.send(DeployImage { - task_package: task_package.clone(), - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; + + Ok(()) +} + +#[cfg_attr(not(feature = "system-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(false); + + let dir = temp_dir!("deploy-restart")?; + let temp_dir = dir.path(); + + log::debug!("Creating directories in: {}", temp_dir.display()); + let work_dir = temp_dir.join("work_dir"); + let cache_dir = temp_dir.join("cache_dir"); + let sub_dir = temp_dir.join("sub_dir"); + + for dir in [work_dir.clone(), cache_dir.clone(), sub_dir.clone()] { + std::fs::create_dir_all(dir)?; + } + + let chunk_size = 4096_usize; + let chunk_count = 1024; + let file_size = (chunk_size * chunk_count) as u64; + let hash = generate_random_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + + log::debug!("Starting HTTP servers"); + let path = temp_dir.to_path_buf(); + start_http(ctx, path) + .await + .expect("unable to start http servers"); + + let task_package = format!( + "hash://sha3:{}:http://127.0.0.1:8001/rnd", + hex::encode(hash) + ); + + log::debug!("Starting TransferService"); + let exe_ctx = TransferServiceContext { + work_dir: work_dir.clone(), + cache_dir, + ..TransferServiceContext::default() + }; + let addr = TransferService::new(exe_ctx).start(); + + log::info!("[>>] Deployment with hash verification"); + let (tx, mut rx) = mpsc::channel::(15); + let mut msg = DeployImage::with_package(&task_package); + msg.forward_progress( + &ProgressArgs::default(), + tx.sink_map_err(|e| ya_transfer::error::Error::Other(e.to_string())), + ); + + tokio::task::spawn_local(async move { + let _result = addr.send(msg).await??; + log::info!("Deployment stopped"); + anyhow::Ok(()) + }); + + let mut last_progress = 0u64; + while let Some(progress) = rx.next().await { + assert_eq!(progress.progress.1.unwrap(), file_size); + assert!(progress.progress.0 >= last_progress); + + last_progress = progress.progress.0; + + log::info!( + "Progress: {}/{}", + progress.progress.0, + progress.progress.1.unwrap_or(0) + ); + } Ok(()) } diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index 69e864f2d7..12d2dc8f92 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -38,6 +38,7 @@ async fn interrupted_transfer( from: src.to_owned(), to: dest.to_owned(), args: TransferArgs::default(), + progress_config: None, }) .await?; @@ -49,7 +50,7 @@ async fn interrupted_transfer( Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_abort(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 4e635249b4..0152509cd0 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -9,7 +9,7 @@ use url::Url; use ya_client_model::activity::TransferArgs; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::hash::verify_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; @@ -112,6 +112,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, + progress_config: None, }) .await??; @@ -126,7 +127,7 @@ async fn transfer( transfer_with_args(addr, from, to, TransferArgs::default()).await } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_resume(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -160,7 +161,7 @@ async fn test_transfer_resume(ctx: &mut DroppableTestContext) -> anyhow::Result< }]; addr.send(AddVolumes::new(volumes)).await??; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 3 * 1024); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 3 * 1024); log::debug!("Starting HTTP servers"); start_http(ctx, temp_dir.to_path_buf()) diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index faa4b136d4..44a315f2c9 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -5,7 +5,7 @@ use test_context::test_context; use ya_client_model::activity::TransferArgs; use ya_exe_unit::error::Error; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::hash::verify_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; @@ -31,13 +31,14 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, + progress_config: None, }) .await??; Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -69,7 +70,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu }, // Uncomment to enable logs ]; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); log::debug!("Starting HTTP servers"); @@ -78,10 +79,10 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -96,18 +97,14 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu println!(); log::warn!("[>>] Deployment with hash verification"); - addr.send(DeployImage { - task_package: task_package.clone(), - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; log::warn!("Deployment complete"); println!(); log::warn!("[>>] Deployment from cache"); - addr.send(DeployImage { - task_package: task_package.clone(), - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; log::warn!("Deployment from cache complete"); println!(); @@ -154,7 +151,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu } #[ignore] -#[cfg_attr(not(feature = "framework-test"), ignore)] +//#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -185,7 +182,7 @@ async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Resul path: "/extract".into(), }, ]; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); log::debug!("Starting HTTP servers"); diff --git a/exe-unit/runtime-api/Cargo.toml b/exe-unit/runtime-api/Cargo.toml index 9de7a43bfc..316a74f2fb 100644 --- a/exe-unit/runtime-api/Cargo.toml +++ b/exe-unit/runtime-api/Cargo.toml @@ -8,12 +8,7 @@ license = "GPL-3.0" homepage = "https://github.com/golemfactory/yagna/tree/master/exe-unit/runtime-api" repository = "https://github.com/golemfactory/yagna" -[[example]] -name = "runtime-server-mock" -required-features = ["server"] - [features] -default = ['server'] codec = [] server = ['prost', 'futures', 'tokio', 'tokio-util'] diff --git a/exe-unit/src/bin.rs b/exe-unit/src/bin.rs index 15c6ede35b..95fb3e5822 100644 --- a/exe-unit/src/bin.rs +++ b/exe-unit/src/bin.rs @@ -1,191 +1,7 @@ -use actix::{Actor, Addr}; -use anyhow::{bail, Context}; -use futures::channel::oneshot; -use std::convert::TryFrom; -use std::path::PathBuf; -use structopt::{clap, StructOpt}; +use structopt::StructOpt; -use ya_client_model::activity::ExeScriptCommand; -use ya_service_bus::RpcEnvelope; - -use ya_core_model::activity; -use ya_exe_unit::agreement::Agreement; use ya_exe_unit::logger::*; -use ya_exe_unit::manifest::ManifestContext; -use ya_exe_unit::message::{GetState, GetStateResponse, Register}; -use ya_exe_unit::runtime::process::RuntimeProcess; -use ya_exe_unit::service::counters; -use ya_exe_unit::service::signal::SignalMonitor; -use ya_exe_unit::state::Supervision; -use ya_exe_unit::{ExeUnit, ExeUnitContext}; -use ya_transfer::transfer::TransferService; -use ya_utils_path::normalize_path; - -#[derive(structopt::StructOpt, Debug)] -#[structopt(global_setting = clap::AppSettings::ColoredHelp)] -#[structopt(version = ya_compile_time_utils::version_describe!())] -struct Cli { - /// Runtime binary path - #[structopt(long, short)] - binary: PathBuf, - #[structopt(flatten)] - supervise: SuperviseCli, - /// Additional runtime arguments - #[structopt( - long, - short, - set = clap::ArgSettings::Global, - number_of_values = 1, - )] - runtime_arg: Vec, - /// Enclave secret key used in secure communication - #[structopt( - long, - env = "EXE_UNIT_SEC_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, - )] - #[allow(dead_code)] - sec_key: Option, - /// Requestor public key used in secure communication - #[structopt( - long, - env = "EXE_UNIT_REQUESTOR_PUB_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, - )] - #[allow(dead_code)] - requestor_pub_key: Option, - #[structopt(subcommand)] - command: Command, -} - -#[derive(structopt::StructOpt, Debug)] -struct SuperviseCli { - /// Hardware resources are handled by the runtime - #[structopt( - long = "runtime-managed-hardware", - alias = "cap-handoff", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, - )] - hardware: bool, - /// Images are handled by the runtime - #[structopt( - long = "runtime-managed-image", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, - )] - image: bool, -} - -#[derive(structopt::StructOpt, Debug)] -#[structopt(global_setting = clap::AppSettings::DeriveDisplayOrder)] -enum Command { - /// Execute commands from file - FromFile { - /// ExeUnit daemon GSB URL - #[structopt(long)] - report_url: Option, - /// ExeUnit service ID - #[structopt(long)] - service_id: Option, - /// Command file path - input: PathBuf, - #[structopt(flatten)] - args: RunArgs, - }, - /// Bind to Service Bus - ServiceBus { - /// ExeUnit service ID - service_id: String, - /// ExeUnit daemon GSB URL - report_url: String, - #[structopt(flatten)] - args: RunArgs, - }, - /// Print an offer template in JSON format - OfferTemplate, - /// Run runtime's test command - Test, -} - -#[derive(structopt::StructOpt, Debug)] -struct RunArgs { - /// Agreement file path - #[structopt(long, short)] - agreement: PathBuf, - /// Working directory - #[structopt(long, short)] - work_dir: PathBuf, - /// Common cache directory - #[structopt(long, short)] - cache_dir: PathBuf, -} - -fn create_path(path: &PathBuf) -> anyhow::Result { - if let Err(error) = std::fs::create_dir_all(path) { - match &error.kind() { - std::io::ErrorKind::AlreadyExists => (), - _ => bail!("Can't create directory: {}, {}", path.display(), error), - } - } - Ok(normalize_path(path)?) -} - -#[cfg(feature = "sgx")] -fn init_crypto( - sec_key: Option, - req_key: Option, -) -> anyhow::Result { - use ya_exe_unit::crypto::Crypto; - - let req_key = req_key.ok_or_else(|| anyhow::anyhow!("Missing requestor public key"))?; - match sec_key { - Some(key) => Ok(Crypto::try_with_keys(key, req_key)?), - None => { - log::info!("Generating a new key pair..."); - Ok(Crypto::try_new(req_key)?) - } - } -} - -async fn send_script( - exe_unit: Addr>, - activity_id: Option, - exe_script: Vec, -) { - use std::time::Duration; - use ya_exe_unit::state::{State, StatePair}; - - let delay = Duration::from_secs_f32(0.5); - loop { - match exe_unit.send(GetState).await { - Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, - Ok(GetStateResponse(StatePair(State::Terminated, _))) - | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) - | Err(_) => { - return log::error!("ExeUnit has terminated"); - } - _ => tokio::time::sleep(delay).await, - } - } - - log::debug!("Executing commands: {:?}", exe_script); - - let msg = activity::Exec { - activity_id: activity_id.unwrap_or_default(), - batch_id: hex::encode(rand::random::<[u8; 16]>()), - exe_script, - timeout: None, - }; - if let Err(e) = exe_unit - .send(RpcEnvelope::with_caller(String::new(), msg)) - .await - { - log::error!("Unable to execute exe script: {:?}", e); - } -} +use ya_exe_unit::{run, Cli}; #[cfg(feature = "packet-trace-enable")] fn init_packet_trace() -> anyhow::Result<()> { @@ -197,147 +13,6 @@ fn init_packet_trace() -> anyhow::Result<()> { Ok(()) } -async fn run() -> anyhow::Result<()> { - dotenv::dotenv().ok(); - - #[cfg(feature = "packet-trace-enable")] - init_packet_trace()?; - - #[allow(unused_mut)] - let mut cli: Cli = Cli::from_args(); - if !cli.binary.exists() { - bail!("Runtime binary does not exist: {}", cli.binary.display()); - } - - let mut commands = None; - let ctx_activity_id; - let ctx_report_url; - - let args = match &cli.command { - Command::FromFile { - args, - service_id, - report_url, - input, - } => { - let contents = std::fs::read_to_string(input).map_err(|e| { - anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) - })?; - let contents = serde_json::from_str(&contents).map_err(|e| { - anyhow::anyhow!( - "Cannot deserialize commands from file {}: {e}", - input.display(), - ) - })?; - ctx_activity_id = service_id.clone(); - ctx_report_url = report_url.clone(); - commands = Some(contents); - args - } - Command::ServiceBus { - args, - service_id, - report_url, - } => { - ctx_activity_id = Some(service_id.clone()); - ctx_report_url = Some(report_url.clone()); - args - } - Command::OfferTemplate => { - let args = cli.runtime_arg.clone(); - let offer_template = ExeUnit::::offer_template(cli.binary, args)?; - println!("{}", serde_json::to_string(&offer_template)?); - return Ok(()); - } - Command::Test => { - let args = cli.runtime_arg.clone(); - let output = ExeUnit::::test(cli.binary, args)?; - println!("{}", String::from_utf8_lossy(&output.stdout)); - eprintln!("{}", String::from_utf8_lossy(&output.stderr)); - if !output.status.success() { - bail!("Test failed"); - } - return Ok(()); - } - }; - - if !args.agreement.exists() { - bail!( - "Agreement file does not exist: {}", - args.agreement.display() - ); - } - let work_dir = create_path(&args.work_dir).map_err(|e| { - anyhow::anyhow!( - "Cannot create the working directory {}: {e}", - args.work_dir.display(), - ) - })?; - let cache_dir = create_path(&args.cache_dir).map_err(|e| { - anyhow::anyhow!( - "Cannot create the cache directory {}: {e}", - args.work_dir.display(), - ) - })?; - let mut agreement = Agreement::try_from(&args.agreement).map_err(|e| { - anyhow::anyhow!( - "Error parsing the agreement from {}: {e}", - args.agreement.display(), - ) - })?; - - log::info!("Attempting to read app manifest .."); - - let manifest_ctx = - ManifestContext::try_new(&agreement.inner).context("Invalid app manifest")?; - agreement.task_package = manifest_ctx - .payload() - .or_else(|| agreement.task_package.take()); - - log::info!("Manifest-enabled features: {:?}", manifest_ctx.features()); - log::info!("User-provided payload: {:?}", agreement.task_package); - - let ctx = ExeUnitContext { - supervise: Supervision { - hardware: cli.supervise.hardware, - image: cli.supervise.image, - manifest: manifest_ctx, - }, - activity_id: ctx_activity_id.clone(), - report_url: ctx_report_url, - agreement, - work_dir, - cache_dir, - runtime_args: cli.runtime_arg.clone(), - acl: Default::default(), - credentials: None, - #[cfg(feature = "sgx")] - crypto: init_crypto( - cli.sec_key.replace("".into()), - cli.requestor_pub_key.clone(), - )?, - }; - - log::debug!("CLI args: {:?}", cli); - log::debug!("ExeUnitContext args: {:?}", ctx); - - let (tx, rx) = oneshot::channel(); - - let counters = counters::build(&ctx, Some(10000), ctx.supervise.hardware).start(); - let transfers = TransferService::new((&ctx).into()).start(); - let runtime = RuntimeProcess::new(&ctx, cli.binary).start(); - let exe_unit = ExeUnit::new(tx, ctx, counters, transfers, runtime).start(); - let signals = SignalMonitor::new(exe_unit.clone()).start(); - exe_unit.send(Register(signals)).await?; - - if let Some(exe_script) = commands { - tokio::task::spawn(send_script(exe_unit, ctx_activity_id, exe_script)); - } - - rx.await??; - Ok(()) -} - #[actix_rt::main] async fn main() { let panic_hook = std::panic::take_hook(); @@ -351,7 +26,16 @@ async fn main() { log::warn!("Using fallback logging due to an error: {:?}", error); }; - std::process::exit(match run().await { + dotenv::dotenv().ok(); + + #[cfg(feature = "packet-trace-enable")] + if let Err(error) = init_packet_trace() { + log::warn!("Initializing packet tracing failed: {error:?}"); + } + + let cli: Cli = Cli::from_args(); + + std::process::exit(match run(cli).await { Ok(_) => 0, Err(error) => { log::error!("{}", error); diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs new file mode 100644 index 0000000000..5b8936ace3 --- /dev/null +++ b/exe-unit/src/exe_unit.rs @@ -0,0 +1,615 @@ +use actix::dev::IntervalFunc; +use actix::{ + Actor, ActorFutureExt, ActorStreamExt, Addr, AsyncContext, Context, ContextFutureSpawner, + Handler, Message, ResponseFuture, Running, StreamHandler, WrapFuture, +}; +use chrono::Utc; +use futures::channel::{mpsc, oneshot}; +use futures::{FutureExt, SinkExt}; +use std::path::PathBuf; +use std::time::Duration; +use tokio::sync::broadcast; +use ya_counters::error::CounterError; +use ya_counters::message::GetCounters; +use ya_counters::service::CountersService; + +use ya_agreement_utils::OfferTemplate; +use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; +use ya_core_model::activity; +use ya_core_model::activity::local::Credentials; +use ya_runtime_api::deploy; +use ya_runtime_api::deploy::ContainerVolume; +use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; +use ya_transfer::transfer::{ + AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, + TransferServiceContext, +}; + +use crate::acl::Acl; +use crate::agreement::Agreement; +use crate::error::Error; +use crate::message::{ + ExecuteCommand, GetStdOut, Initialize, RuntimeEvent, SetState, Shutdown, ShutdownReason, + SignExeScript, Stop, UpdateDeployment, +}; +use crate::runtime::{Runtime, RuntimeMode}; +use crate::service::{self, ServiceAddr, ServiceControl}; +use crate::state::{ExeUnitState, StateError, Supervision}; +use crate::Result; + +lazy_static::lazy_static! { + static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); +} + +#[derive(Clone, Debug, Default, Message)] +#[rtype(result = "Result>")] +pub struct FinishNotifier {} + +pub struct ExeUnit { + pub(crate) ctx: ExeUnitContext, + pub(crate) state: ExeUnitState, + pub(crate) events: Channel, + pub(crate) runtime: Addr, + pub(crate) counters: Addr, + pub(crate) transfers: Addr, + pub(crate) services: Vec>, + pub(crate) shutdown_tx: broadcast::Sender<()>, +} + +impl ExeUnit { + pub fn new( + ctx: ExeUnitContext, + counters: Addr, + transfers: Addr, + runtime: Addr, + ) -> Self { + let (shutdown_tx, _) = broadcast::channel(1); + ExeUnit { + ctx, + state: ExeUnitState::default(), + events: Channel::default(), + runtime: runtime.clone(), + counters: counters.clone(), + transfers: transfers.clone(), + services: vec![ + Box::new(ServiceAddr::new(counters)), + Box::new(ServiceAddr::new(transfers)), + Box::new(ServiceAddr::new(runtime)), + ], + shutdown_tx, + } + } + + pub fn offer_template(binary: PathBuf, args: Vec) -> crate::Result { + use crate::runtime::process::RuntimeProcess; + + let runtime_template = RuntimeProcess::offer_template(binary, args)?; + let supervisor_template = OfferTemplate::new(serde_json::json!({ + "golem.com.usage.vector": service::counters::usage_vector(), + "golem.activity.caps.transfer.protocol": TransferService::schemes(), + "golem.activity.caps.transfer.report-progress": true, + "golem.activity.caps.deploy.report-progress": true, + })); + + Ok(supervisor_template.patch(runtime_template)) + } + + pub fn test(binary: PathBuf, args: Vec) -> crate::Result { + use crate::runtime::process::RuntimeProcess; + RuntimeProcess::test(binary, args) + } + + fn report_usage(&mut self, context: &mut Context) { + if self.ctx.activity_id.is_none() || self.ctx.report_url.is_none() { + return; + } + let fut = report_usage( + self.ctx.report_url.clone().unwrap(), + self.ctx.activity_id.clone().unwrap(), + context.address(), + self.counters.clone(), + ); + context.spawn(fut.into_actor(self)); + } + + pub(crate) async fn stop_runtime(runtime: Addr, reason: ShutdownReason) { + if let Err(e) = runtime + .send(Shutdown(reason)) + .timeout(Duration::from_secs(5u64)) + .await + { + log::warn!("Unable to stop the runtime: {:?}", e); + } + } +} + +#[derive(Clone)] +pub struct RuntimeRef(Addr>); + +impl RuntimeRef { + pub fn from_ctx(ctx: &Context>) -> Self { + RuntimeRef(ctx.address()) + } +} + +impl std::ops::Deref for RuntimeRef { + type Target = Addr>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl RuntimeRef { + pub async fn exec( + self, + exec: activity::Exec, + runtime: Addr, + transfers: Addr, + mut events: mpsc::Sender, + mut control: oneshot::Receiver<()>, + ) { + let batch_id = exec.batch_id.clone(); + for (idx, command) in exec.exe_script.into_iter().enumerate() { + if let Ok(Some(_)) = control.try_recv() { + log::warn!("Batch {} execution aborted", batch_id); + break; + } + + let runtime_cmd = ExecuteCommand { + batch_id: batch_id.clone(), + command: command.clone(), + tx: events.clone(), + idx, + }; + + let evt = RuntimeEvent::started(batch_id.clone(), idx, command.clone()); + if let Err(e) = events.send(evt).await { + log::error!("Unable to report event: {:?}", e); + } + + let (return_code, message) = match { + if runtime_cmd.stateless() { + self.exec_stateless(&runtime_cmd).await + } else { + self.exec_stateful(runtime_cmd, &runtime, &transfers).await + } + } { + Ok(_) => (0, None), + Err(ref err) => match err { + Error::CommandExitCodeError(c) => (*c, Some(err.to_string())), + _ => (-1, Some(err.to_string())), + }, + }; + + let evt = RuntimeEvent::finished(batch_id.clone(), idx, return_code, message.clone()); + if let Err(e) = events.send(evt).await { + log::error!("Unable to report event: {:?}", e); + } + + if return_code != 0 { + let message = message.unwrap_or_else(|| "reason unspecified".into()); + log::warn!("Batch {} execution interrupted: {}", batch_id, message); + break; + } + } + } + + async fn exec_stateless(&self, runtime_cmd: &ExecuteCommand) -> crate::Result<()> { + match runtime_cmd.command { + ExeScriptCommand::Sign {} => { + let batch_id = runtime_cmd.batch_id.clone(); + let signature = self.send(SignExeScript { batch_id }).await??; + let stdout = serde_json::to_string(&signature)?; + + runtime_cmd + .tx + .clone() + .send(RuntimeEvent::stdout( + runtime_cmd.batch_id.clone(), + runtime_cmd.idx, + CommandOutput::Bin(stdout.into_bytes()), + )) + .await + .map_err(|e| Error::runtime(format!("Unable to send stdout event: {:?}", e)))?; + } + ExeScriptCommand::Terminate {} => { + log::debug!("Terminating running ExeScripts"); + let exclude_batches = vec![runtime_cmd.batch_id.clone()]; + self.send(Stop { exclude_batches }).await??; + self.send(SetState::from(State::Initialized)).await?; + } + _ => (), + } + Ok(()) + } + + async fn exec_stateful( + &self, + runtime_cmd: ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + let state = self.send(crate::message::GetState {}).await?.0; + let state_pre = match (&state.0, &state.1) { + (_, Some(_)) => { + return Err(StateError::Busy(state).into()); + } + (State::New, _) | (State::Terminated, _) => { + return Err(StateError::InvalidState(state).into()); + } + (State::Initialized, _) => match &runtime_cmd.command { + ExeScriptCommand::Deploy { .. } => { + StatePair(State::Initialized, Some(State::Deployed)) + } + _ => return Err(StateError::InvalidState(state).into()), + }, + (State::Deployed, _) => match &runtime_cmd.command { + ExeScriptCommand::Start { .. } => StatePair(State::Deployed, Some(State::Ready)), + _ => return Err(StateError::InvalidState(state).into()), + }, + (s, _) => match &runtime_cmd.command { + ExeScriptCommand::Deploy { .. } | ExeScriptCommand::Start { .. } => { + return Err(StateError::InvalidState(state).into()); + } + _ => StatePair(*s, Some(*s)), + }, + }; + self.send(SetState::from(state_pre)).await?; + + log::info!("Executing command: {:?}", runtime_cmd.command); + + let result = async { + self.pre_runtime(&runtime_cmd, runtime, transfer_service) + .await?; + + let exit_code = runtime.send(runtime_cmd.clone()).await??; + if exit_code != 0 { + return Err(Error::CommandExitCodeError(exit_code)); + } + + self.post_runtime(&runtime_cmd, runtime, transfer_service) + .await?; + + Ok(()) + } + .await; + + let state_cur = self.send(crate::message::GetState {}).await?.0; + if state_cur != state_pre { + return Err(StateError::UnexpectedState { + current: state_cur, + expected: state_pre, + } + .into()); + } + + self.send(SetState::from(state_pre.1.unwrap())).await?; + result + } + + async fn pre_runtime( + &self, + runtime_cmd: &ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + match &runtime_cmd.command { + ExeScriptCommand::Transfer { + from, + to, + args, + progress, + } => { + let mut msg = TransferResource { + from: from.clone(), + to: to.clone(), + args: args.clone(), + progress_config: None, + }; + + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + transfer_service.send(msg).await??; + } + ExeScriptCommand::Deploy { + net, + hosts, + progress, + volumes, + .. + } => { + let volumes = volumes + .iter() + .enumerate() + .map(|(idx, vol)| ContainerVolume { + name: format!("vol-custom-{idx}"), + path: vol.to_string(), + }) + .collect::>(); + transfer_service.send(AddVolumes::new(volumes)).await??; + + // TODO: We should pass `task_package` here not in `TransferService` initialization. + let mut msg = DeployImage::default(); + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + + let task_package = transfer_service.send(msg).await??; + runtime + .send(UpdateDeployment { + task_package, + networks: Some(net.clone()), + hosts: Some(hosts.clone()), + ..Default::default() + }) + .await??; + } + _ => (), + } + Ok(()) + } + + async fn post_runtime( + &self, + runtime_cmd: &ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + if let ExeScriptCommand::Deploy { .. } = &runtime_cmd.command { + let mut runtime_mode = RuntimeMode::ProcessPerCommand; + let stdout = self + .send(GetStdOut { + batch_id: runtime_cmd.batch_id.clone(), + idx: runtime_cmd.idx, + }) + .await?; + + if let Some(output) = stdout { + let deployment = deploy::DeployResult::from_bytes(output).map_err(|e| { + log::error!("Deployment failed: {}", e); + Error::CommandError(e.to_string()) + })?; + transfer_service + .send(AddVolumes::new(deployment.vols)) + .await??; + runtime_mode = deployment.start_mode.into(); + } + runtime + .send(UpdateDeployment { + runtime_mode: Some(runtime_mode), + ..Default::default() + }) + .await??; + } + Ok(()) + } +} + +impl Actor for ExeUnit { + type Context = Context; + + fn started(&mut self, ctx: &mut Self::Context) { + let rx = self.events.rx.take().unwrap(); + Self::add_stream(rx, ctx); + + let addr = ctx.address(); + if let Some(activity_id) = &self.ctx.activity_id { + let srv_id = activity::exeunit::bus_id(activity_id); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + + #[cfg(feature = "sgx")] + { + actix_rpc::bind::( + &srv_id, + addr.clone().recipient(), + ); + } + #[cfg(not(feature = "sgx"))] + { + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::binds::( + &srv_id, + addr.clone().recipient(), + ); + } + } + + IntervalFunc::new(*DEFAULT_REPORT_INTERVAL, Self::report_usage) + .finish() + .spawn(ctx); + + log::info!("Initializing manifests"); + self.ctx + .supervise + .manifest + .build_validators() + .into_actor(self) + .map(|result, this, ctx| match result { + Ok(validators) => { + this.ctx.supervise.manifest.add_validators(validators); + log::info!("Manifest initialization complete"); + } + Err(e) => { + let err = Error::Other(format!("manifest initialization error: {}", e)); + log::error!("Supervisor is shutting down due to {}", err); + ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); + } + }) + .wait(ctx); + + let addr_ = addr.clone(); + async move { + addr.send(Initialize).await?.map_err(Error::from)?; + addr.send(SetState::from(State::Initialized)).await?; + Ok::<_, Error>(()) + } + .then(|result| async move { + match result { + Ok(_) => log::info!("Supervisor initialized"), + Err(e) => { + let err = Error::Other(format!("initialization error: {}", e)); + log::error!("Supervisor is shutting down due to {}", err); + let _ = addr_.send(Shutdown(ShutdownReason::Error(err))).await; + } + } + }) + .into_actor(self) + .spawn(ctx); + } + + fn stopping(&mut self, _: &mut Self::Context) -> Running { + if self.state.inner.0 == State::Terminated { + return Running::Stop; + } + Running::Continue + } + + fn stopped(&mut self, _: &mut Self::Context) { + self.shutdown_tx.send(()).ok(); + } +} + +#[derive(derivative::Derivative)] +#[derivative(Debug)] +pub struct ExeUnitContext { + pub supervise: Supervision, + pub activity_id: Option, + pub report_url: Option, + pub agreement: Agreement, + pub work_dir: PathBuf, + pub cache_dir: PathBuf, + pub runtime_args: Vec, + pub acl: Acl, + pub credentials: Option, + #[cfg(feature = "sgx")] + #[derivative(Debug = "ignore")] + pub crypto: crate::crypto::Crypto, +} + +impl ExeUnitContext { + pub fn verify_activity_id(&self, activity_id: &str) -> crate::Result<()> { + match &self.activity_id { + Some(act_id) => match act_id == activity_id { + true => Ok(()), + false => Err(Error::RemoteServiceError(format!( + "Forbidden! Invalid activity id: {}", + activity_id + ))), + }, + None => Ok(()), + } + } +} + +impl From<&ExeUnitContext> for TransferServiceContext { + fn from(val: &ExeUnitContext) -> Self { + TransferServiceContext { + task_package: val.agreement.task_package.clone(), + deploy_retry: None, + cache_dir: val.cache_dir.clone(), + work_dir: val.work_dir.clone(), + transfer_retry: None, + } + } +} + +pub struct Channel { + pub(crate) tx: mpsc::Sender, + rx: Option>, +} + +impl Default for Channel { + fn default() -> Self { + let (tx, rx) = mpsc::channel(8); + Channel { tx, rx: Some(rx) } + } +} + +pub async fn report(url: S, msg: M) -> bool +where + M: RpcMessage + Unpin + 'static, + S: AsRef, +{ + let url = url.as_ref(); + match ya_service_bus::typed::service(url).send(msg).await { + Err(ya_service_bus::Error::Timeout(msg)) => { + log::warn!("Timed out reporting to {}: {}", url, msg); + true + } + Err(e) => { + log::error!("Error reporting to {}: {:?}", url, e); + false + } + Ok(Err(e)) => { + log::error!("Error response while reporting to {}: {:?}", url, e); + false + } + Ok(Ok(_)) => true, + } +} + +async fn report_usage( + report_url: String, + activity_id: String, + exe_unit: Addr>, + metrics: Addr, +) { + match metrics.send(GetCounters).await { + Ok(resp) => match resp { + Ok(data) => { + let msg = activity::local::SetUsage { + activity_id, + usage: ActivityUsage { + current_usage: Some(data), + timestamp: Utc::now().timestamp(), + }, + timeout: None, + }; + if !report(&report_url, msg).await { + exe_unit.do_send(Shutdown(ShutdownReason::Error(Error::RuntimeError( + format!("Reporting endpoint '{}' is not available", report_url), + )))); + } + } + Err(err) => match err { + CounterError::UsageLimitExceeded(info) => { + log::warn!("Usage limit exceeded: {}", info); + exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); + } + error => log::warn!("Unable to retrieve metrics: {:?}", error), + }, + }, + Err(e) => log::warn!("Unable to report activity usage: {:?}", e), + } +} + +impl Handler for ExeUnit { + type Result = Result>; + + fn handle(&mut self, _msg: FinishNotifier, _: &mut Self::Context) -> Self::Result { + Ok(self.shutdown_tx.subscribe()) + } +} + +impl Handler for TransferService { + type Result = ResponseFuture>; + + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + let addr = ctx.address(); + async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() + } +} + +impl Handler for CountersService { + type Result = ResponseFuture>; + + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + let addr = ctx.address(); + async move { Ok(addr.send(ya_counters::message::Shutdown {}).await??) }.boxed_local() + } +} diff --git a/exe-unit/src/handlers/local.rs b/exe-unit/src/handlers/local.rs index b737234415..1ca55591e5 100644 --- a/exe-unit/src/handlers/local.rs +++ b/exe-unit/src/handlers/local.rs @@ -1,11 +1,12 @@ +use actix::prelude::*; +use futures::FutureExt; + use crate::error::Error; use crate::message::*; use crate::runtime::Runtime; use crate::service::ServiceAddr; use crate::state::State; use crate::{report, ExeUnit}; -use actix::prelude::*; -use futures::FutureExt; use ya_client_model::activity; use ya_core_model::activity::local::SetState as SetActivityState; diff --git a/exe-unit/src/handlers/rpc.rs b/exe-unit/src/handlers/rpc.rs index db06fd93f0..a343b9573c 100644 --- a/exe-unit/src/handlers/rpc.rs +++ b/exe-unit/src/handlers/rpc.rs @@ -23,6 +23,7 @@ impl Handler> for ExeUnit { type Result = as Message>::Result; fn handle(&mut self, msg: RpcEnvelope, ctx: &mut Self::Context) -> Self::Result { + log::debug!("Received Exec message: {:?}", msg.as_ref()); self.ctx.verify_activity_id(&msg.activity_id)?; let batch_id = msg.batch_id.clone(); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 90c63adff1..d76bac574a 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -1,36 +1,26 @@ #[macro_use] extern crate derive_more; +use actix::prelude::*; +use anyhow::{bail, Context}; +use std::convert::TryFrom; use std::path::PathBuf; -use std::time::Duration; +use structopt::clap; +use ya_counters::service::CountersServiceBuilder; -use actix::prelude::*; -use chrono::Utc; -use futures::channel::{mpsc, oneshot}; -use futures::{FutureExt, SinkExt}; - -use ya_agreement_utils::agreement::OfferTemplate; -use ya_client_model::activity::{ - activity_state::StatePair, ActivityUsage, CommandOutput, ExeScriptCommand, State, -}; +use ya_client_model::activity::ExeScriptCommand; use ya_core_model::activity; -use ya_core_model::activity::local::Credentials; -use ya_counters::error::CounterError; -use ya_counters::message::GetCounters; -use ya_counters::service::CountersService; -use ya_runtime_api::deploy; -use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; -use ya_transfer::transfer::{ - AddVolumes, DeployImage, TransferResource, TransferService, TransferServiceContext, -}; - -use crate::acl::Acl; +use ya_service_bus::RpcEnvelope; +use ya_transfer::transfer::TransferService; +use ya_utils_path::normalize_path; + use crate::agreement::Agreement; use crate::error::Error; -use crate::message::*; -use crate::runtime::*; -use crate::service::{ServiceAddr, ServiceControl}; -use crate::state::{ExeUnitState, StateError, Supervision}; +use crate::manifest::ManifestContext; +use crate::message::{GetState, GetStateResponse, Register}; +use crate::runtime::process::RuntimeProcess; +use crate::service::signal::SignalMonitor; +use crate::state::Supervision; mod acl; pub mod agreement; @@ -49,540 +39,344 @@ pub mod service; pub mod state; mod dns; +mod exe_unit; + +pub use exe_unit::{report, ExeUnit, ExeUnitContext, FinishNotifier, RuntimeRef}; + pub type Result = std::result::Result; -lazy_static::lazy_static! { - static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); +#[derive(structopt::StructOpt, Debug)] +#[structopt(global_setting = clap::AppSettings::ColoredHelp)] +#[structopt(version = ya_compile_time_utils::version_describe!())] +pub struct Cli { + /// Runtime binary path + #[structopt(long, short)] + pub binary: PathBuf, + #[structopt(flatten)] + pub supervise: SuperviseCli, + /// Additional runtime arguments + #[structopt( + long, + short, + set = clap::ArgSettings::Global, + number_of_values = 1, + )] + pub runtime_arg: Vec, + /// Enclave secret key used in secure communication + #[structopt( + long, + env = "EXE_UNIT_SEC_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, + )] + #[allow(dead_code)] + pub sec_key: Option, + /// Requestor public key used in secure communication + #[structopt( + long, + env = "EXE_UNIT_REQUESTOR_PUB_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, + )] + #[allow(dead_code)] + pub requestor_pub_key: Option, + #[structopt(subcommand)] + pub command: Command, } -pub struct ExeUnit { - ctx: ExeUnitContext, - state: ExeUnitState, - events: Channel, - runtime: Addr, - counters: Addr, - transfers: Addr, - services: Vec>, - shutdown_tx: Option>>, +#[derive(structopt::StructOpt, Debug, Clone)] +pub struct SuperviseCli { + /// Hardware resources are handled by the runtime + #[structopt( + long = "runtime-managed-hardware", + alias = "cap-handoff", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, + )] + pub hardware: bool, + /// Images are handled by the runtime + #[structopt( + long = "runtime-managed-image", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, + )] + pub image: bool, } -impl ExeUnit { - pub fn new( - shutdown_tx: oneshot::Sender>, - ctx: ExeUnitContext, - counters: Addr, - transfers: Addr, - runtime: Addr, - ) -> Self { - ExeUnit { - ctx, - state: ExeUnitState::default(), - events: Channel::default(), - runtime: runtime.clone(), - counters: counters.clone(), - transfers: transfers.clone(), - services: vec![ - Box::new(ServiceAddr::new(counters)), - Box::new(ServiceAddr::new(transfers)), - Box::new(ServiceAddr::new(runtime)), - ], - shutdown_tx: Some(shutdown_tx), - } - } - - pub fn offer_template(binary: PathBuf, args: Vec) -> Result { - use crate::runtime::process::RuntimeProcess; - - let runtime_template = RuntimeProcess::offer_template(binary, args)?; - let supervisor_template = OfferTemplate::new(serde_json::json!({ - "golem.com.usage.vector": service::counters::usage_vector(), - "golem.activity.caps.transfer.protocol": TransferService::schemes(), - })); - - Ok(supervisor_template.patch(runtime_template)) - } - - pub fn test(binary: PathBuf, args: Vec) -> Result { - use crate::runtime::process::RuntimeProcess; - RuntimeProcess::test(binary, args) - } - - fn report_usage(&mut self, context: &mut Context) { - if self.ctx.activity_id.is_none() || self.ctx.report_url.is_none() { - return; - } - let fut = report_usage( - self.ctx.report_url.clone().unwrap(), - self.ctx.activity_id.clone().unwrap(), - context.address(), - self.counters.clone(), - ); - context.spawn(fut.into_actor(self)); - } - - async fn stop_runtime(runtime: Addr, reason: ShutdownReason) { - if let Err(e) = runtime - .send(Shutdown(reason)) - .timeout(Duration::from_secs(5u64)) - .await - { - log::warn!("Unable to stop the runtime: {:?}", e); - } - } +#[derive(structopt::StructOpt, Debug)] +#[structopt(global_setting = clap::AppSettings::DeriveDisplayOrder)] +pub enum Command { + /// Execute commands from file + FromFile { + /// ExeUnit daemon GSB URL + #[structopt(long)] + report_url: Option, + /// ExeUnit service ID + #[structopt(long)] + service_id: Option, + /// Command file path + input: PathBuf, + #[structopt(flatten)] + args: RunArgs, + }, + /// Bind to Service Bus + ServiceBus { + /// ExeUnit service ID + service_id: String, + /// ExeUnit daemon GSB URL + report_url: String, + #[structopt(flatten)] + args: RunArgs, + }, + /// Print an offer template in JSON format + OfferTemplate, + /// Run runtime's test command + Test, } -#[derive(Clone)] -struct RuntimeRef(Addr>); - -impl RuntimeRef { - fn from_ctx(ctx: &Context>) -> Self { - RuntimeRef(ctx.address()) - } +#[derive(structopt::StructOpt, Debug, Clone)] +pub struct RunArgs { + /// Agreement file path + #[structopt(long, short)] + pub agreement: PathBuf, + /// Working directory + #[structopt(long, short)] + pub work_dir: PathBuf, + /// Common cache directory + #[structopt(long, short)] + pub cache_dir: PathBuf, } -impl std::ops::Deref for RuntimeRef { - type Target = Addr>; - - fn deref(&self) -> &Self::Target { - &self.0 +fn create_path(path: &PathBuf) -> anyhow::Result { + if let Err(error) = std::fs::create_dir_all(path) { + match &error.kind() { + std::io::ErrorKind::AlreadyExists => (), + _ => bail!("Can't create directory: {}, {}", path.display(), error), + } } + Ok(normalize_path(path)?) } -impl RuntimeRef { - async fn exec( - self, - exec: activity::Exec, - runtime: Addr, - transfers: Addr, - mut events: mpsc::Sender, - mut control: oneshot::Receiver<()>, - ) { - let batch_id = exec.batch_id.clone(); - for (idx, command) in exec.exe_script.into_iter().enumerate() { - if let Ok(Some(_)) = control.try_recv() { - log::warn!("Batch {} execution aborted", batch_id); - break; - } - - let runtime_cmd = ExecuteCommand { - batch_id: batch_id.clone(), - command: command.clone(), - tx: events.clone(), - idx, - }; - - let evt = RuntimeEvent::started(batch_id.clone(), idx, command.clone()); - if let Err(e) = events.send(evt).await { - log::error!("Unable to report event: {:?}", e); - } - - let (return_code, message) = match { - if runtime_cmd.stateless() { - self.exec_stateless(&runtime_cmd).await - } else { - self.exec_stateful(runtime_cmd, &runtime, &transfers).await - } - } { - Ok(_) => (0, None), - Err(ref err) => match err { - Error::CommandExitCodeError(c) => (*c, Some(err.to_string())), - _ => (-1, Some(err.to_string())), - }, - }; - - let evt = RuntimeEvent::finished(batch_id.clone(), idx, return_code, message.clone()); - if let Err(e) = events.send(evt).await { - log::error!("Unable to report event: {:?}", e); - } - - if return_code != 0 { - let message = message.unwrap_or_else(|| "reason unspecified".into()); - log::warn!("Batch {} execution interrupted: {}", batch_id, message); - break; - } +#[cfg(feature = "sgx")] +fn init_crypto( + sec_key: Option, + req_key: Option, +) -> anyhow::Result { + let req_key = req_key.ok_or_else(|| anyhow::anyhow!("Missing requestor public key"))?; + match sec_key { + Some(key) => Ok(crate::crypto::Crypto::try_with_keys(key, req_key)?), + None => { + log::info!("Generating a new key pair..."); + Ok(crate::crypto::Crypto::try_new(req_key)?) } } +} - async fn exec_stateless(&self, runtime_cmd: &ExecuteCommand) -> Result<()> { - match runtime_cmd.command { - ExeScriptCommand::Sign {} => { - let batch_id = runtime_cmd.batch_id.clone(); - let signature = self.send(SignExeScript { batch_id }).await??; - let stdout = serde_json::to_string(&signature)?; - - runtime_cmd - .tx - .clone() - .send(RuntimeEvent::stdout( - runtime_cmd.batch_id.clone(), - runtime_cmd.idx, - CommandOutput::Bin(stdout.into_bytes()), - )) - .await - .map_err(|e| Error::runtime(format!("Unable to send stdout event: {:?}", e)))?; - } - ExeScriptCommand::Terminate {} => { - log::debug!("Terminating running ExeScripts"); - let exclude_batches = vec![runtime_cmd.batch_id.clone()]; - self.send(Stop { exclude_batches }).await??; - self.send(SetState::from(State::Initialized)).await?; +pub async fn send_script( + exe_unit: Addr>, + activity_id: Option, + exe_script: Vec, +) -> anyhow::Result { + use crate::state::{State, StatePair}; + use std::time::Duration; + + let delay = Duration::from_secs_f32(0.5); + loop { + match exe_unit.send(GetState).await { + Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, + Ok(GetStateResponse(StatePair(State::Terminated, _))) + | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) + | Err(_) => { + log::error!("ExeUnit has terminated"); + bail!("ExeUnit has terminated"); } - _ => (), + _ => tokio::time::sleep(delay).await, } - Ok(()) } - async fn exec_stateful( - &self, - runtime_cmd: ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - let state = self.send(GetState {}).await?.0; - let state_pre = match (&state.0, &state.1) { - (_, Some(_)) => { - return Err(StateError::Busy(state).into()); - } - (State::New, _) | (State::Terminated, _) => { - return Err(StateError::InvalidState(state).into()); - } - (State::Initialized, _) => match &runtime_cmd.command { - ExeScriptCommand::Deploy { .. } => { - StatePair(State::Initialized, Some(State::Deployed)) - } - _ => return Err(StateError::InvalidState(state).into()), - }, - (State::Deployed, _) => match &runtime_cmd.command { - ExeScriptCommand::Start { .. } => StatePair(State::Deployed, Some(State::Ready)), - _ => return Err(StateError::InvalidState(state).into()), - }, - (s, _) => match &runtime_cmd.command { - ExeScriptCommand::Deploy { .. } | ExeScriptCommand::Start { .. } => { - return Err(StateError::InvalidState(state).into()); - } - _ => StatePair(*s, Some(*s)), - }, - }; - self.send(SetState::from(state_pre)).await?; - - log::info!("Executing command: {:?}", runtime_cmd.command); - - let result = async { - self.pre_runtime(&runtime_cmd, runtime, transfer_service) - .await?; - - let exit_code = runtime.send(runtime_cmd.clone()).await??; - if exit_code != 0 { - return Err(Error::CommandExitCodeError(exit_code)); - } + log::debug!("Executing commands: {:?}", exe_script); - self.post_runtime(&runtime_cmd, runtime, transfer_service) - .await?; + let batch_id = hex::encode(rand::random::<[u8; 16]>()); + let msg = activity::Exec { + activity_id: activity_id.unwrap_or_default(), + batch_id: batch_id.clone(), + exe_script, + timeout: None, + }; - Ok(()) - } - .await; - - let state_cur = self.send(GetState {}).await?.0; - if state_cur != state_pre { - return Err(StateError::UnexpectedState { - current: state_cur, - expected: state_pre, - } - .into()); - } + exe_unit + .send(RpcEnvelope::with_caller(String::new(), msg)) + .await??; + Ok(batch_id) +} - self.send(SetState::from(state_pre.1.unwrap())).await?; - result - } +// We need this mut for conditional compilation for sgx +#[allow(unused_mut)] +pub async fn run(mut cli: Cli) -> anyhow::Result<()> { + log::debug!("CLI args: {:?}", cli); - async fn pre_runtime( - &self, - runtime_cmd: &ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - match &runtime_cmd.command { - ExeScriptCommand::Transfer { from, to, args } => { - let msg = TransferResource { - from: from.clone(), - to: to.clone(), - args: args.clone(), - }; - transfer_service.send(msg).await??; - } - ExeScriptCommand::Deploy { net, hosts } => { - // TODO: We should pass `task_package` here not in `TransferService` initialization. - let task_package = transfer_service - .send(DeployImage { task_package: None }) - .await??; - runtime - .send(UpdateDeployment { - task_package, - networks: Some(net.clone()), - hosts: Some(hosts.clone()), - ..Default::default() - }) - .await??; - } - _ => (), - } - Ok(()) + if !cli.binary.exists() { + bail!("Runtime binary does not exist: {}", cli.binary.display()); } - async fn post_runtime( - &self, - runtime_cmd: &ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - if let ExeScriptCommand::Deploy { .. } = &runtime_cmd.command { - let mut runtime_mode = RuntimeMode::ProcessPerCommand; - let stdout = self - .send(GetStdOut { - batch_id: runtime_cmd.batch_id.clone(), - idx: runtime_cmd.idx, - }) - .await?; - - if let Some(output) = stdout { - let deployment = deploy::DeployResult::from_bytes(output).map_err(|e| { - log::error!("Deployment failed: {}", e); - Error::CommandError(e.to_string()) - })?; - transfer_service - .send(AddVolumes::new(deployment.vols)) - .await??; - runtime_mode = deployment.start_mode.into(); - } - runtime - .send(UpdateDeployment { - runtime_mode: Some(runtime_mode), - ..Default::default() - }) - .await??; + let mut commands = None; + let ctx_activity_id; + let ctx_report_url; + + let args = match cli.command { + Command::FromFile { + args, + service_id, + report_url, + input, + } => { + let contents = std::fs::read_to_string(&input).map_err(|e| { + anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) + })?; + let contents = serde_json::from_str(&contents).map_err(|e| { + anyhow::anyhow!( + "Cannot deserialize commands from file {}: {e}", + input.display(), + ) + })?; + ctx_activity_id = service_id.clone(); + ctx_report_url = report_url.clone(); + commands = Some(contents); + args } - Ok(()) - } -} - -impl Actor for ExeUnit { - type Context = Context; - - fn started(&mut self, ctx: &mut Self::Context) { - let rx = self.events.rx.take().unwrap(); - Self::add_stream(rx, ctx); - - let addr = ctx.address(); - if let Some(activity_id) = &self.ctx.activity_id { - let srv_id = activity::exeunit::bus_id(activity_id); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - - #[cfg(feature = "sgx")] - { - actix_rpc::bind::( - &srv_id, - addr.clone().recipient(), - ); - } - #[cfg(not(feature = "sgx"))] - { - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::binds::( - &srv_id, - addr.clone().recipient(), - ); - } + Command::ServiceBus { + args, + service_id, + report_url, + } => { + ctx_activity_id = Some(service_id.clone()); + ctx_report_url = Some(report_url.clone()); + args } - - IntervalFunc::new(*DEFAULT_REPORT_INTERVAL, Self::report_usage) - .finish() - .spawn(ctx); - - log::info!("Initializing manifests"); - self.ctx - .supervise - .manifest - .build_validators() - .into_actor(self) - .map(|result, this, ctx| match result { - Ok(validators) => { - this.ctx.supervise.manifest.add_validators(validators); - log::info!("Manifest initialization complete"); - } - Err(e) => { - let err = Error::Other(format!("manifest initialization error: {}", e)); - log::error!("Supervisor is shutting down due to {}", err); - ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); - } - }) - .wait(ctx); - - let addr_ = addr.clone(); - async move { - addr.send(Initialize).await?.map_err(Error::from)?; - addr.send(SetState::from(State::Initialized)).await?; - Ok::<_, Error>(()) + Command::OfferTemplate => { + let args = cli.runtime_arg.clone(); + let offer_template = ExeUnit::::offer_template(cli.binary, args)?; + println!("{}", serde_json::to_string(&offer_template)?); + return Ok(()); } - .then(|result| async move { - match result { - Ok(_) => log::info!("Supervisor initialized"), - Err(e) => { - let err = Error::Other(format!("initialization error: {}", e)); - log::error!("Supervisor is shutting down due to {}", err); - let _ = addr_.send(Shutdown(ShutdownReason::Error(err))).await; - } + Command::Test => { + let args = cli.runtime_arg.clone(); + let output = ExeUnit::::test(cli.binary, args)?; + println!("{}", String::from_utf8_lossy(&output.stdout)); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + if !output.status.success() { + bail!("Test failed"); } - }) - .into_actor(self) - .spawn(ctx); - } - - fn stopping(&mut self, _: &mut Self::Context) -> Running { - if self.state.inner.0 == State::Terminated { - return Running::Stop; + return Ok(()); } - Running::Continue + }; + + let exe_unit = exe_unit(ExeUnitConfig { + report_url: ctx_report_url, + service_id: ctx_activity_id.clone(), + runtime_args: cli.runtime_arg, + binary: cli.binary, + supervise: cli.supervise, + sec_key: cli.sec_key, + args, + requestor_pub_key: cli.requestor_pub_key, + }) + .await?; + + if let Some(exe_script) = commands { + tokio::task::spawn(send_script(exe_unit.clone(), ctx_activity_id, exe_script)); } - fn stopped(&mut self, _: &mut Self::Context) { - if let Some(tx) = self.shutdown_tx.take() { - let _ = tx.send(Ok(())); - } - } + exe_unit.send(FinishNotifier {}).await??.recv().await?; + Ok(()) } -#[derive(derivative::Derivative)] -#[derivative(Debug)] -pub struct ExeUnitContext { - pub supervise: Supervision, - pub activity_id: Option, - pub report_url: Option, - pub agreement: Agreement, - pub work_dir: PathBuf, - pub cache_dir: PathBuf, +#[derive(Debug, Clone)] +pub struct ExeUnitConfig { + pub args: RunArgs, + pub binary: PathBuf, pub runtime_args: Vec, - pub acl: Acl, - pub credentials: Option, - #[cfg(feature = "sgx")] - #[derivative(Debug = "ignore")] - pub crypto: crypto::Crypto, -} - -impl ExeUnitContext { - pub fn verify_activity_id(&self, activity_id: &str) -> Result<()> { - match &self.activity_id { - Some(act_id) => match act_id == activity_id { - true => Ok(()), - false => Err(Error::RemoteServiceError(format!( - "Forbidden! Invalid activity id: {}", - activity_id - ))), - }, - None => Ok(()), - } - } -} - -impl From<&ExeUnitContext> for TransferServiceContext { - fn from(val: &ExeUnitContext) -> Self { - TransferServiceContext { - task_package: val.agreement.task_package.clone(), - deploy_retry: None, - cache_dir: val.cache_dir.clone(), - work_dir: val.work_dir.clone(), - transfer_retry: None, - } - } -} - -struct Channel { - tx: mpsc::Sender, - rx: Option>, -} + pub service_id: Option, + pub report_url: Option, + pub supervise: SuperviseCli, -impl Default for Channel { - fn default() -> Self { - let (tx, rx) = mpsc::channel(8); - Channel { tx, rx: Some(rx) } - } + #[allow(dead_code)] + pub sec_key: Option, + #[allow(dead_code)] + pub requestor_pub_key: Option, } -pub(crate) async fn report(url: S, msg: M) -> bool -where - M: RpcMessage + Unpin + 'static, - S: AsRef, -{ - let url = url.as_ref(); - match ya_service_bus::typed::service(url).send(msg).await { - Err(ya_service_bus::Error::Timeout(msg)) => { - log::warn!("Timed out reporting to {}: {}", url, msg); - true - } - Err(e) => { - log::error!("Error reporting to {}: {:?}", url, e); - false - } - Ok(Err(e)) => { - log::error!("Error response while reporting to {}: {:?}", url, e); - false - } - Ok(Ok(_)) => true, +// Mut is necessary in case of sgx compilation :((((( +#[allow(unused_mut)] +pub async fn exe_unit(mut config: ExeUnitConfig) -> anyhow::Result>> { + let args = config.args; + if !args.agreement.exists() { + bail!( + "Agreement file does not exist: {}", + args.agreement.display() + ); } -} - -async fn report_usage( - report_url: String, - activity_id: String, - exe_unit: Addr>, - counters: Addr, -) { - match counters.send(GetCounters).await { - Ok(resp) => match resp { - Ok(data) => { - let msg = activity::local::SetUsage { - activity_id, - usage: ActivityUsage { - current_usage: Some(data), - timestamp: Utc::now().timestamp(), - }, - timeout: None, - }; - if !report(&report_url, msg).await { - exe_unit.do_send(Shutdown(ShutdownReason::Error(Error::RuntimeError( - format!("Reporting endpoint '{}' is not available", report_url), - )))); - } - } - Err(err) => match err { - CounterError::UsageLimitExceeded(info) => { - log::warn!("Usage limit exceeded: {}", info); - exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); - } - error => log::warn!("Unable to retrieve counters: {:?}", error), - }, + let work_dir = create_path(&args.work_dir).map_err(|e| { + anyhow::anyhow!( + "Cannot create the working directory {}: {e}", + args.work_dir.display(), + ) + })?; + let cache_dir = create_path(&args.cache_dir).map_err(|e| { + anyhow::anyhow!( + "Cannot create the cache directory {}: {e}", + args.work_dir.display(), + ) + })?; + let mut agreement = Agreement::try_from(&args.agreement).map_err(|e| { + anyhow::anyhow!( + "Error parsing the agreement from {}: {e}", + args.agreement.display(), + ) + })?; + + log::info!("Attempting to read app manifest .."); + + let manifest_ctx = + ManifestContext::try_new(&agreement.inner).context("Invalid app manifest")?; + agreement.task_package = manifest_ctx + .payload() + .or_else(|| agreement.task_package.take()); + + log::info!("Manifest-enabled features: {:?}", manifest_ctx.features()); + log::info!("User-provided payload: {:?}", agreement.task_package); + + let ctx = ExeUnitContext { + supervise: Supervision { + hardware: config.supervise.hardware, + image: config.supervise.image, + manifest: manifest_ctx, }, - Err(e) => log::warn!("Unable to report activity usage: {:?}", e), - } -} - -impl Handler for TransferService { - type Result = ResponseFuture>; - - fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { - let addr = ctx.address(); - async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() - } -} - -impl Handler for CountersService { - type Result = ResponseFuture>; - - fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { - let addr = ctx.address(); - async move { Ok(addr.send(ya_counters::message::Shutdown {}).await??) }.boxed_local() - } + activity_id: config.service_id.clone(), + report_url: config.report_url, + agreement, + work_dir, + cache_dir, + runtime_args: config.runtime_args, + acl: Default::default(), + credentials: None, + #[cfg(feature = "sgx")] + crypto: init_crypto( + config.sec_key.replace("".into()), + config.requestor_pub_key.clone(), + )?, + }; + + log::debug!("ExeUnitContext args: {:?}", ctx); + + let metrics = CountersServiceBuilder::new(ctx.agreement.usage_vector.clone(), Some(1000)) + .build() + .start(); + let transfers = TransferService::new((&ctx).into()).start(); + let runtime = RuntimeProcess::new(&ctx, config.binary).start(); + let exe_unit = ExeUnit::new(ctx, metrics, transfers, runtime).start(); + let signals = SignalMonitor::new(exe_unit.clone()).start(); + exe_unit.send(Register(signals)).await?; + + Ok(exe_unit) } diff --git a/exe-unit/src/logger.rs b/exe-unit/src/logger.rs index d9cd42565d..9d36cd3473 100644 --- a/exe-unit/src/logger.rs +++ b/exe-unit/src/logger.rs @@ -6,7 +6,7 @@ const ENV_VAR_LOG_DIR: &str = "EXE_UNIT_LOG_DIR"; const ENV_VAR_FILE_LOG_LEVEL: &str = "EXE_UNIT_FILE_LOG_LEVEL"; const DEFAULT_LOG_LEVEL: &str = "info"; -const DEFAULT_FILE_LOG_LEVEL: &str = "debug"; +const DEFAULT_FILE_LOG_LEVEL: &str = "debug,tokio_util=off,h2=info"; const DEFAULT_LOG_DIR: &str = "logs"; const DEFAULT_LOG_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.3f%z"; diff --git a/exe-unit/src/manifest.rs b/exe-unit/src/manifest.rs index 300bcd4e56..f662a0838e 100644 --- a/exe-unit/src/manifest.rs +++ b/exe-unit/src/manifest.rs @@ -503,6 +503,10 @@ mod tests { ExeScriptCommand::Deploy { net: Default::default(), hosts: Default::default(), + hostname: None, + volumes: vec![], + env: Default::default(), + progress: None, }, ExeScriptCommand::Start { args: Default::default(), @@ -657,6 +661,7 @@ mod tests { from: "/src/0x0add".to_string(), to: "/dst/0x0add".to_string(), args: Default::default(), + progress: None, }]; let validator: ScriptValidator = r#"{ diff --git a/exe-unit/src/message.rs b/exe-unit/src/message.rs index bf2bd473e2..d6125b29b3 100644 --- a/exe-unit/src/message.rs +++ b/exe-unit/src/message.rs @@ -5,6 +5,7 @@ use crate::Result; use actix::prelude::*; use futures::channel::mpsc; +use futures::{Sink, SinkExt}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::PathBuf; @@ -12,7 +13,9 @@ use std::path::PathBuf; use ya_client_model::activity; use ya_client_model::activity::activity_state::{State, StatePair}; use ya_client_model::activity::exe_script_command::Network; -use ya_client_model::activity::{CommandOutput, ExeScriptCommand, ExeScriptCommandResult}; +use ya_client_model::activity::{ + CommandOutput, CommandProgress, ExeScriptCommand, ExeScriptCommandResult, +}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "GetStateResponse")] @@ -99,8 +102,19 @@ impl ExecuteCommand { }, ) } + + pub fn progress_sink(&self) -> impl Sink { + let CommandContext { batch_id, idx, .. } = self.clone().split().1; + self.tx.clone().with(move |item| { + let batch_id = batch_id.clone(); + futures::future::ok(RuntimeEvent::Process(activity::RuntimeEvent::progress( + batch_id, idx, item, + ))) + }) + } } +#[allow(clippy::large_enum_variant)] #[derive(Clone, Debug)] pub enum RuntimeEvent { Process(activity::RuntimeEvent), diff --git a/exe-unit/src/state.rs b/exe-unit/src/state.rs index 5729a1416f..e0e59c80ce 100644 --- a/exe-unit/src/state.rs +++ b/exe-unit/src/state.rs @@ -164,6 +164,7 @@ impl Batch { ..event }) } + RuntimeEventKind::Progress(_) => Some(event), }; if let Some(evt) = stream_event { diff --git a/exe-unit/tests/resources/agreement.json b/exe-unit/tests/resources/agreement.json new file mode 100644 index 0000000000..427dbee17f --- /dev/null +++ b/exe-unit/tests/resources/agreement.json @@ -0,0 +1,80 @@ +{ + "agreementId": "0ec929f5acc8f98a47ab72d61a2c2f343d45d8438d3aa4ccdc84e717c219e185", + "proposedSignature": "NoSignature", + "state": "Pending", + "timestamp": "2022-05-22T10:41:42.564784259Z", + "validTo": "2022-05-22T11:41:42.562457Z", + + "offer": { + "properties": { + "golem.activity.caps.transfer.protocol": [ + "gftp", + "https", + "http" + ], + "golem.activity.caps.transfer.report-progress": true, + "golem.com.payment.debit-notes.accept-timeout?": 240, + "golem.com.payment.platform.erc20-goerli-tglm.address": "0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a", + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": [ + 0.001388888888888889, + 0.0002777777777777778, + 0.0 + ], + "golem.com.scheme": "payu", + "golem.com.scheme.payu.debit-note.interval-sec?": 120, + "golem.com.scheme.payu.payment-timeout-sec?": 120, + "golem.com.usage.vector": [ + "golem.usage.cpu_sec", + "golem.usage.duration_sec" + ], + "golem.inf.cpu.architecture": "x86_64", + "golem.inf.cpu.brand": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz", + "golem.inf.cpu.capabilities": [], + "golem.inf.cpu.cores": 6, + "golem.inf.cpu.model": "Stepping 10 Family 6 Model 302", + "golem.inf.cpu.threads": 11, + "golem.inf.cpu.vendor": "GenuineIntel", + "golem.inf.mem.gib": 28.0, + "golem.inf.storage.gib": 10.188121795654297, + "golem.node.id.name": "mock-provider", + "golem.node.net.is-public": true, + "golem.runtime.capabilities": [], + "golem.runtime.name": "ya-mock-runtime", + "golem.runtime.version": "0.1.0", + "golem.srv.caps.multi-activity": true, + "golem.srv.caps.payload-manifest": true + }, + "constraints": "(&\n (golem.srv.comp.expiration>1705586871777)\n)", + "offerId": "afce49b1ea5b45db91bdd6e5481479f9-9095fca9dea0a91ce95cf994125b33cdd838fcc963a1106f2be9e4b5b65a52f0", + "providerId": "0x86a269498fb5270f20bdc6fdcf6039122b0d3b23", + "timestamp": "2022-05-22T10:41:42.564784259Z" + }, + + "demand": { + "constraints": "(&(golem.com.payment.platform.erc20-goerli-tglm.address=*)\n\t(golem.com.pricing.model=linear)\n\t(&(golem.inf.mem.gib>=0.5)\n\t(golem.inf.storage.gib>=2.0)\n\t(golem.inf.cpu.threads>=1)\n\t(golem.runtime.name=ya-mock-runtime)))", + "demandId": "773035fc685c46da8e61473ac2a2568e-3f3eb86d6ef9a01708d0f57d0b19cc69fd74422150c120e33cc1b5f4a1a12b96", + "properties": { + "golem": { + "com.payment": { + "chosen-platform": "erc20-goerli-tglm", + "debit-notes.accept-timeout?": 240, + "platform.erc20-goerli-tglm.address": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7" + }, + "node": {}, + "srv": { + "caps.multi-activity": true, + "comp": { + "expiration": 1653216996555, + "task_package": "hash://sha3:22e08b990b6c6685a7e80ecd9a1adb52561a7d9fe9e69b915269da229be6c1ad69dea4ff8a77dc2c4973558da9150909a2be4121b1cbe1ddb04630c1f75aad4f:http://127.0.0.1:8001/image-1", + "vm": { + "package_format": "gvmkit-squash" + } + } + } + } + }, + "requestorId": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7", + "timestamp": "2022-05-22T10:41:42.564784259Z" + } +} diff --git a/exe-unit/tests/resources/agreement.template.json b/exe-unit/tests/resources/agreement.template.json new file mode 100644 index 0000000000..a3488f8354 --- /dev/null +++ b/exe-unit/tests/resources/agreement.template.json @@ -0,0 +1,80 @@ +{ + "agreementId": "0ec929f5acc8f98a47ab72d61a2c2f343d45d8438d3aa4ccdc84e717c219e185", + "proposedSignature": "NoSignature", + "state": "Pending", + "timestamp": "2022-05-22T10:41:42.564784259Z", + "validTo": "2022-05-22T11:41:42.562457Z", + + "offer": { + "properties": { + "golem.activity.caps.transfer.protocol": [ + "gftp", + "https", + "http" + ], + "golem.activity.caps.transfer.report-progress": true, + "golem.com.payment.debit-notes.accept-timeout?": 240, + "golem.com.payment.platform.erc20-goerli-tglm.address": "0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a", + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": [ + 0.001388888888888889, + 0.0002777777777777778, + 0.0 + ], + "golem.com.scheme": "payu", + "golem.com.scheme.payu.debit-note.interval-sec?": 120, + "golem.com.scheme.payu.payment-timeout-sec?": 120, + "golem.com.usage.vector": [ + "golem.usage.cpu_sec", + "golem.usage.duration_sec" + ], + "golem.inf.cpu.architecture": "x86_64", + "golem.inf.cpu.brand": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz", + "golem.inf.cpu.capabilities": [], + "golem.inf.cpu.cores": 6, + "golem.inf.cpu.model": "Stepping 10 Family 6 Model 302", + "golem.inf.cpu.threads": 11, + "golem.inf.cpu.vendor": "GenuineIntel", + "golem.inf.mem.gib": 28.0, + "golem.inf.storage.gib": 10.188121795654297, + "golem.node.id.name": "mock-provider", + "golem.node.net.is-public": true, + "golem.runtime.capabilities": [], + "golem.runtime.name": "ya-mock-runtime", + "golem.runtime.version": "0.1.0", + "golem.srv.caps.multi-activity": true, + "golem.srv.caps.payload-manifest": true + }, + "constraints": "(&\n (golem.srv.comp.expiration>1705586871777)\n)", + "offerId": "afce49b1ea5b45db91bdd6e5481479f9-9095fca9dea0a91ce95cf994125b33cdd838fcc963a1106f2be9e4b5b65a52f0", + "providerId": "0x86a269498fb5270f20bdc6fdcf6039122b0d3b23", + "timestamp": "2022-05-22T10:41:42.564784259Z" + }, + + "demand": { + "constraints": "(&(golem.com.payment.platform.erc20-goerli-tglm.address=*)\n\t(golem.com.pricing.model=linear)\n\t(&(golem.inf.mem.gib>=0.5)\n\t(golem.inf.storage.gib>=2.0)\n\t(golem.inf.cpu.threads>=1)\n\t(golem.runtime.name=ya-mock-runtime)))", + "demandId": "773035fc685c46da8e61473ac2a2568e-3f3eb86d6ef9a01708d0f57d0b19cc69fd74422150c120e33cc1b5f4a1a12b96", + "properties": { + "golem": { + "com.payment": { + "chosen-platform": "erc20-goerli-tglm", + "debit-notes.accept-timeout?": 240, + "platform.erc20-goerli-tglm.address": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7" + }, + "node": {}, + "srv": { + "caps.multi-activity": true, + "comp": { + "expiration": 1653216996555, + "task_package": "${task-package}", + "vm": { + "package_format": "gvmkit-squash" + } + } + } + } + }, + "requestorId": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7", + "timestamp": "2022-05-22T10:41:42.564784259Z" + } +} diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs new file mode 100644 index 0000000000..5c354e4b6c --- /dev/null +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -0,0 +1,53 @@ +use test_context::test_context; + +use ya_client_model::activity::ExeScriptCommand; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::file::generate_image; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::cargo_binary; +use ya_framework_basic::{resource, temp_dir}; +use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; + +#[cfg_attr(not(feature = "system-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(false); + + let dir = temp_dir!("exe-unit-start-terminate")?; + let temp_dir = dir.path(); + let image_repo = temp_dir.join("images"); + + generate_image(&image_repo, "image-1", 4096_usize, 10); + start_http(ctx, image_repo) + .await + .expect("unable to start http servers"); + + let config = exe_unit_config( + temp_dir, + &resource!("agreement.json"), + cargo_binary("ya-mock-runtime")?, + ); + + let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + exe.await_init().await.unwrap(); + + log::info!("Sending [deploy, start] batch for execution."); + + exe.wait_for_batch(&exe.deploy(None).await.unwrap()) + .await + .unwrap(); + exe.wait_for_batch(&exe.start(vec![]).await.unwrap()) + .await + .unwrap(); + + log::info!("Sending shutdown request."); + + exe.exec(None, vec![ExeScriptCommand::Terminate {}]) + .await + .unwrap(); + + exe.shutdown().await.unwrap(); + Ok(()) +} diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs new file mode 100644 index 0000000000..c87bbe4394 --- /dev/null +++ b/exe-unit/tests/test_progress.rs @@ -0,0 +1,151 @@ +use futures::StreamExt; +use std::time::Duration; +use test_context::test_context; + +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::{ExeScriptCommand, RuntimeEventKind, TransferArgs}; +use ya_core_model::activity; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::file::generate_image; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::{cargo_binary, template}; +use ya_framework_basic::{resource, temp_dir}; +use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; + +use ya_service_bus::typed as bus; + +/// Test if progress reporting mechanisms work on gsb level +/// with full ExeUnit setup. +#[cfg_attr(not(feature = "system-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("progress-reporting")?; + let temp_dir = dir.path(); + let image_repo = temp_dir.join("images"); + + let chunk_size = 4096_usize; + let chunk_count = 1024 * 20; + let file_size = (chunk_size * chunk_count) as u64; + + let hash = generate_image(&image_repo, "image-big", chunk_size, chunk_count); + let package = format!( + "hash://sha3:{}:http://127.0.0.1:8001/image-big", + hex::encode(hash) + ); + start_http(ctx, image_repo.clone()) + .await + .expect("unable to start http servers"); + + let config = exe_unit_config( + temp_dir, + &template( + &resource!("agreement.template.json"), + temp_dir.join("agreement.json"), + &[("task-package", package)], + )?, + cargo_binary("ya-mock-runtime")?, + ); + + let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + exe.await_init().await.unwrap(); + + log::info!("Sending [deploy] batch for execution."); + + let batch_id = exe + .exec( + None, + vec![ExeScriptCommand::Deploy { + net: vec![], + progress: Some(ProgressArgs { + update_interval: Some(Duration::from_millis(300)), + update_step: None, + }), + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec!["/input".to_owned()], + }], + ) + .await + .unwrap(); + + validate_progress( + config.service_id.clone().unwrap(), + batch_id.clone(), + file_size, + ) + .await; + + exe.wait_for_batch(&batch_id).await.unwrap(); + exe.wait_for_batch(&exe.start(vec![]).await.unwrap()) + .await + .unwrap(); + + let batch_id = exe + .exec( + None, + vec![ExeScriptCommand::Transfer { + args: TransferArgs::default(), + progress: Some(ProgressArgs { + update_interval: Some(Duration::from_millis(100)), + update_step: None, + }), + // Important: Use hashed transfer, because it is significantly slower in debug mode. + // Otherwise we won't get any progress message, because it is too fast. + from: format!( + "hash://sha3:{}:http://127.0.0.1:8001/image-big", + hex::encode(hash) + ), + to: "container:/input/image-copy".to_string(), + }], + ) + .await + .unwrap(); + + validate_progress(config.service_id.unwrap(), batch_id.clone(), file_size).await; + exe.wait_for_batch(&batch_id).await.unwrap(); + Ok(()) +} + +async fn validate_progress(activity_id: String, batch_id: String, file_size: u64) { + let msg = activity::StreamExecBatchResults { + activity_id: activity_id.clone(), + batch_id: batch_id.clone(), + }; + + // Note: Since we have already sent commands, we may loose a few events on the beginning. + // Our API has a problem here. We can't call `StreamExecBatchResults` before Exeunit knows + // `batch_id`. Even if we would generate id ourselves (possible in test, but not possible for Requestor), + // we still can't call this function too early. + let mut stream = bus::service(activity::exeunit::bus_id(&msg.activity_id)).call_streaming(msg); + + let mut last_progress = 0u64; + let mut num_progresses = 0u64; + while let Some(Ok(Ok(item))) = stream.next().await { + if item.index == 0 { + match item.kind { + RuntimeEventKind::Finished { return_code, .. } => { + assert_eq!(return_code, 0); + break; + } + RuntimeEventKind::Progress(progress) => { + log::info!("Progress report: {:?}", progress); + + assert_eq!(progress.step, (0, 1)); + assert_eq!(progress.unit, Some("Bytes".to_string())); + assert_eq!(progress.progress.1.unwrap(), file_size); + assert!(progress.progress.0 >= last_progress); + + last_progress = progress.progress.0; + num_progresses += 1; + } + _ => (), + } + } + } + assert!(num_progresses > 1); +} diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index 1cb3f35556..1c884aef97 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -35,6 +35,7 @@ async-compression = { version = "=0.3.7", features = [ "xz", ] } bytes = "1.0" +cargo_metadata = "0.18" crossterm = "0.26.1" env_logger = "0.7" futures = "0.3.4" @@ -51,6 +52,7 @@ serde = "1.0.104" sha2 = "0.8.1" sha3 = "0.8.2" tempdir = "0.3.7" +test-binary = { version = "3.0", git = "https://github.com/golemfactory/test-binary.git" } test-context = "0.1.4" thiserror = "1.0.11" tokio = { version = "1", features = ["fs", "io-util"] } diff --git a/test-utils/test-framework/framework-basic/src/file.rs b/test-utils/test-framework/framework-basic/src/file.rs index 8d05e872e6..969e1dff5a 100644 --- a/test-utils/test-framework/framework-basic/src/file.rs +++ b/test-utils/test-framework/framework-basic/src/file.rs @@ -1,17 +1,66 @@ +use rand::rngs::ThreadRng; use rand::Rng; use sha2::Digest; +use std::fs; use std::fs::OpenOptions; use std::io::Write; use std::path::{Path, PathBuf}; use crate::hash::HashOutput; -pub fn generate_file_with_hash( +trait ContentGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec; +} + +struct RandomGenerator(pub ThreadRng); +struct ZeroGenerator {} + +impl ContentGenerator for RandomGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec { + (0..chunk_size) + .map(|_| self.0.gen_range(0..256) as u8) + .collect() + } +} + +impl ContentGenerator for ZeroGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec { + vec![0; chunk_size] + } +} + +pub fn generate_random_file_with_hash( path: &Path, name: &str, chunk_size: usize, chunk_count: usize, ) -> HashOutput { + generate_file_with_hash_( + path, + name, + chunk_size, + chunk_count, + RandomGenerator(rand::thread_rng()), + ) +} + +pub fn generate_image( + path: &Path, + name: &str, + chunk_size: usize, + chunk_count: usize, +) -> HashOutput { + generate_file_with_hash_(path, name, chunk_size, chunk_count, ZeroGenerator {}) +} + +fn generate_file_with_hash_( + path: &Path, + name: &str, + chunk_size: usize, + chunk_count: usize, + mut gen: impl ContentGenerator, +) -> HashOutput { + fs::create_dir_all(path).ok(); let path = path.join(name); log::debug!( @@ -26,8 +75,6 @@ pub fn generate_file_with_hash( .open(path) .expect("rnd file"); - let mut rng = rand::thread_rng(); - for i in 0..chunk_count { log::trace!( "Generating chunk {i}/{chunk_count}. File size: {}/{}", @@ -35,9 +82,7 @@ pub fn generate_file_with_hash( chunk_count * chunk_size ); - let input: Vec = (0..chunk_size) - .map(|_| rng.gen_range(0..256) as u8) - .collect(); + let input: Vec = gen.generate(chunk_size); hasher.input(&input); let _ = file_src.write(&input).unwrap(); diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs index bbe35b4643..374c697b2a 100644 --- a/test-utils/test-framework/framework-basic/src/test_dirs.rs +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -1,7 +1,9 @@ -use anyhow::anyhow; +use anyhow::{anyhow, bail}; use std::fs; use std::path::{Path, PathBuf}; +use std::str::FromStr; use tempdir::TempDir; +use test_binary::TestBinary; pub mod macros { /// Creates temporary directory in cargo target directory. @@ -33,6 +35,65 @@ pub fn temp_dir_(base_dir: &str, prefix: &str) -> anyhow::Result { Ok(dir) } +#[cfg(debug_assertions)] +pub fn is_debug() -> bool { + true +} + +#[cfg(not(debug_assertions))] +pub fn is_debug() -> bool { + false +} + +#[cfg(target_family = "windows")] +pub fn extension() -> String { + ".exe".to_string() +} + +#[cfg(not(target_family = "windows"))] +pub fn extension() -> String { + "".to_string() +} + +fn find_binary(bin_name: &str) -> anyhow::Result { + let current = std::env::current_exe() + .map_err(|e| anyhow!("Failed to get path to current binary. {e}"))? + .parent() + .and_then(|path| path.parent()) + .ok_or(anyhow!("No parent dir for current binary."))? + .to_path_buf(); + let bin_name = format!("{bin_name}{}", extension()); + let bin_path = current.join(&bin_name); + if !bin_path.exists() { + bail!( + "Path doesn't exist: {}, when looking for binary: {}", + bin_path.display(), + bin_name + ); + } + + if !bin_path.is_file() { + bail!("Expected {} to be binary file.", bin_path.display()); + } + + Ok(bin_path) +} + +/// Returns path to test binary from workspace. +pub fn cargo_binary(bin_name: &str) -> anyhow::Result { + // Check if binary is already compiled. + if find_binary(bin_name).is_err() { + TestBinary::from_workspace(bin_name)? + .build() + .map_err(|e| anyhow!("Failed to compile binary: {e}"))? + .to_str() + .map(PathBuf::from_str) + .ok_or(anyhow!("Failed to convert path from OsString"))??; + }; + + find_binary(bin_name) +} + /// Returns resource from `resources` directory in tests. pub fn resource_(base_dir: &str, name: &str) -> PathBuf { PathBuf::from(base_dir) diff --git a/tests/readme.md b/tests/readme.md index 84919fcc7d..02f4742e73 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -11,7 +11,7 @@ To run all tests including framework tests and unit tests (but without market te `cargo test --workspace --features framework-test` To run only framework tests use command: -`cargo test --test '*' --features framework-test` +`cargo test --test '*' -p yagna -p ya-exe-unit -p ya-transfer --features framework-test` ## Creating tests