diff --git a/.gitmodules b/.gitmodules index 954732f6..5ffec13a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,6 +5,4 @@ [submodule "3rd_party/SplitStream"] path = 3rd_party/SplitStream url = https://github.com/lynzrand/SplitStream.git -[submodule "judger/3rd_party/bollard"] - path = judger/3rd_party/bollard - url = https://github.com/fussybeaver/bollard.git + diff --git a/README.md b/README.md index ff6ff952..7b5758c3 100644 --- a/README.md +++ b/README.md @@ -64,3 +64,7 @@ $ cargo build ``` $ path/to/rurikawa connect ``` + +#### 限制 + +目前不能在 Windows 下运行 diff --git a/judger/3rd_party/bollard b/judger/3rd_party/bollard deleted file mode 160000 index e50137a3..00000000 --- a/judger/3rd_party/bollard +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e50137a37b1b7ddc17a4047cee7f5041418c06a7 diff --git a/judger/Cargo.lock b/judger/Cargo.lock index fd9dbc7a..7cb0f9dd 100644 --- a/judger/Cargo.lock +++ b/judger/Cargo.lock @@ -24,7 +24,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -33,7 +33,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -81,7 +81,7 @@ dependencies = [ "futures-io", "once_cell", "pin-project-lite 0.1.11", - "tokio", + "tokio 0.2.25", ] [[package]] @@ -131,7 +131,7 @@ dependencies = [ "polling", "vec-arena", "waker-fn", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -159,7 +159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3205e9ef663755ece59cd2e57ac2518d8dc7a2b7e86cc0aca04f641cc1566215" dependencies = [ "log", - "tokio", + "tokio 0.2.25", ] [[package]] @@ -175,7 +175,7 @@ dependencies = [ "futures-lite", "once_cell", "signal-hook", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -206,6 +206,27 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "async-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-tar" version = "0.3.0" @@ -251,7 +272,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -260,12 +281,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.0" @@ -337,43 +352,40 @@ dependencies = [ [[package]] name = "bollard" -version = "0.8.0" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699194c00f3a2effd3358d47f880646818e3d483190b17ebcdf598c654fb77e9" dependencies = [ - "base64 0.12.3", + "base64", "bollard-stubs", - "bytes 0.5.6", + "bytes 1.0.1", "chrono", "ct-logs", - "dirs", + "dirs-next", "futures-core", "futures-util", "hex", "http", "hyper", - "hyper-rustls", "hyper-unix-connector", "log", - "mio-named-pipes", - "pin-project 0.4.27", - "rustls", - "rustls-native-certs", + "pin-project 1.0.5", "serde", "serde_derive", "serde_json", - "serde_urlencoded 0.6.1", + "serde_urlencoded", "thiserror", - "tokio", + "tokio 1.2.0", "tokio-util", "url", - "webpki-roots", - "winapi 0.3.9", + "winapi", ] [[package]] name = "bollard-stubs" -version = "1.40.6" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf72b3eeb9a5cce41979def2c7522cb830356c0621ca29c0b766128c4e7fded" +checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" dependencies = [ "chrono", "serde", @@ -390,7 +402,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "parking_lot", + "parking_lot 0.10.2", "slab", ] @@ -462,7 +474,7 @@ dependencies = [ "num-traits", "serde", "time", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -569,32 +581,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" -[[package]] -name = "core-foundation" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" -dependencies = [ - "core-foundation-sys 0.7.0", - "libc", -] - [[package]] name = "core-foundation" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" dependencies = [ - "core-foundation-sys 0.8.2", + "core-foundation-sys", "libc", ] -[[package]] -name = "core-foundation-sys" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" - [[package]] name = "core-foundation-sys" version = "0.8.2" @@ -626,9 +622,9 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "ct-logs" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8e13110a84b6315df212c045be706af261fd364791cad863285439ebba672e" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" dependencies = [ "sct", ] @@ -650,7 +646,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b57a92e9749e10f25a171adcebfafe72991d45e7ec2dcb853e8f83d9dafaeb08" dependencies = [ "nix", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -758,6 +754,16 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.3.5" @@ -765,15 +771,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" dependencies = [ "libc", - "redox_users", - "winapi 0.3.9", + "redox_users 0.3.5", + "winapi", ] [[package]] -name = "dtoa" -version = "0.4.7" +name = "dirs-sys-next" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d7ed2934d741c6b37e33e3832298e8850b53fd2d2bea03873375596c7cea4e" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users 0.4.0", + "winapi", +] [[package]] name = "encoding_rs" @@ -844,7 +855,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.2.4", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -897,22 +908,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "futures" version = "0.3.12" @@ -1023,18 +1018,6 @@ dependencies = [ "slab", ] -[[package]] -name = "futures_codec" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" -dependencies = [ - "bytes 0.5.6", - "futures", - "memchr", - "pin-project 0.4.27", -] - [[package]] name = "generic-array" version = "0.14.4" @@ -1090,11 +1073,11 @@ dependencies = [ [[package]] name = "h2" -version = "0.2.7" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "fnv", "futures-core", "futures-sink", @@ -1102,7 +1085,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio", + "tokio 1.2.0", "tokio-util", "tracing", "tracing-futures", @@ -1151,11 +1134,11 @@ dependencies = [ [[package]] name = "http-body" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "http", ] @@ -1179,11 +1162,11 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.13.10" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" +checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "futures-channel", "futures-core", "futures-util", @@ -1195,55 +1178,36 @@ dependencies = [ "itoa", "pin-project 1.0.5", "socket2", - "tokio", + "tokio 1.2.0", "tower-service", "tracing", "want", ] -[[package]] -name = "hyper-rustls" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" -dependencies = [ - "bytes 0.5.6", - "ct-logs", - "futures-util", - "hyper", - "log", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", - "webpki", -] - [[package]] name = "hyper-tls" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "hyper", "native-tls", - "tokio", - "tokio-tls", + "tokio 1.2.0", + "tokio-native-tls", ] [[package]] name = "hyper-unix-connector" -version = "0.1.5" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42b66be14087ec25c5150c9d1228a1e9bbbfe7fe2506ff85daed350724980319" +checksum = "24ef1fd95d34b4ff007d3f0590727b5cf33572cace09b42032fc817dc8b16557" dependencies = [ "anyhow", - "futures-util", "hex", "hyper", - "pin-project 0.4.27", - "tokio", + "pin-project 1.0.5", + "tokio 1.2.0", ] [[package]] @@ -1275,11 +1239,11 @@ dependencies = [ [[package]] name = "input_buffer" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" +checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", ] [[package]] @@ -1291,15 +1255,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "ipnet" version = "2.3.0" @@ -1321,16 +1276,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "kv-log-macro" version = "1.0.7" @@ -1365,7 +1310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" dependencies = [ "cfg-if 1.0.0", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1423,68 +1368,17 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" -[[package]] -name = "mime_guess" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "mio" -version = "0.6.23" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7" dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", "libc", "log", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - -[[package]] -name = "mio-named-pipes" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" -dependencies = [ - "log", - "mio", - "miow 0.3.6", - "winapi 0.3.9", -] - -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", + "miow", + "ntapi", + "winapi", ] [[package]] @@ -1494,7 +1388,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ "socket2", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1528,8 +1422,8 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework 2.0.0", - "security-framework-sys 2.0.0", + "security-framework", + "security-framework-sys", "tempfile", ] @@ -1540,18 +1434,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8123a81538e457d44b933a02faf885d3fe8408806b23fa700e8f01c6c3a98998" dependencies = [ "libc", - "winapi 0.3.9", -] - -[[package]] -name = "net2" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1576,6 +1459,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1662,7 +1554,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1678,7 +1570,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" dependencies = [ "lock_api 0.3.4", - "parking_lot_core", + "parking_lot_core 0.7.2", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api 0.4.2", + "parking_lot_core 0.8.2", ] [[package]] @@ -1692,7 +1595,21 @@ dependencies = [ "libc", "redox_syscall 0.1.57", "smallvec", - "winapi 0.3.9", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi", ] [[package]] @@ -1805,7 +1722,7 @@ dependencies = [ "libc", "log", "wepoll-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1909,7 +1826,7 @@ dependencies = [ "libc", "rand_core 0.3.1", "rdrand", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2043,6 +1960,16 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom 0.2.2", + "redox_syscall 0.2.4", +] + [[package]] name = "regex" version = "1.4.3" @@ -2083,17 +2010,17 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] name = "reqwest" -version = "0.10.10" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" +checksum = "fd281b1030aa675fb90aa994d07187645bb3c8fc756ca766e7c3070b439de9de" dependencies = [ - "base64 0.13.0", - "bytes 0.5.6", + "base64", + "bytes 1.0.1", "encoding_rs", "futures-core", "futures-util", @@ -2106,15 +2033,14 @@ dependencies = [ "lazy_static", "log", "mime", - "mime_guess", "native-tls", "percent-encoding", "pin-project-lite 0.2.4", "serde", "serde_json", - "serde_urlencoded 0.7.0", - "tokio", - "tokio-tls", + "serde_urlencoded", + "tokio 1.2.0", + "tokio-native-tls", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -2134,7 +2060,7 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2211,7 +2137,6 @@ dependencies = [ "err-derive", "fern", "futures", - "futures_codec", "http", "hyper", "libc", @@ -2229,7 +2154,8 @@ dependencies = [ "serde_json", "shell-words", "tar", - "tokio", + "tokio 1.2.0", + "tokio-stream", "tokio-test", "tokio-tungstenite", "tokio-util", @@ -2238,7 +2164,6 @@ dependencies = [ "tracing-futures", "tracing-log", "tracing-subscriber", - "tungstenite", ] [[package]] @@ -2247,7 +2172,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" dependencies = [ - "base64 0.13.0", + "base64", "blake2b_simd", "constant_time_eq", "crossbeam-utils", @@ -2259,31 +2184,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustls" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" -dependencies = [ - "base64 0.12.3", - "log", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls-native-certs" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629d439a7672da82dd955498445e496ee2096fe2117b9f796558a43fdb9e59b8" -dependencies = [ - "openssl-probe", - "rustls", - "schannel", - "security-framework 1.0.0", -] - [[package]] name = "rustversion" version = "1.0.4" @@ -2303,7 +2203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" dependencies = [ "lazy_static", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2322,19 +2222,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "security-framework" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad502866817f0575705bd7be36e2b2535cc33262d493aa733a2ec862baa2bc2b" -dependencies = [ - "bitflags", - "core-foundation 0.7.0", - "core-foundation-sys 0.7.0", - "libc", - "security-framework-sys 1.0.0", -] - [[package]] name = "security-framework" version = "2.0.0" @@ -2342,20 +2229,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" dependencies = [ "bitflags", - "core-foundation 0.9.1", - "core-foundation-sys 0.8.2", - "libc", - "security-framework-sys 2.0.0", -] - -[[package]] -name = "security-framework-sys" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ceb04988b17b6d1dcd555390fa822ca5637b4a14e1f5099f13d351bed4d6c7" -dependencies = [ - "core-foundation-sys 0.7.0", + "core-foundation", + "core-foundation-sys", "libc", + "security-framework-sys", ] [[package]] @@ -2364,7 +2241,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" dependencies = [ - "core-foundation-sys 0.8.2", + "core-foundation-sys", "libc", ] @@ -2399,18 +2276,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" -dependencies = [ - "dtoa", - "itoa", - "serde", - "url", -] - [[package]] name = "serde_urlencoded" version = "0.7.0" @@ -2518,7 +2383,7 @@ checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" dependencies = [ "cfg-if 1.0.0", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2599,7 +2464,7 @@ dependencies = [ "rand 0.8.3", "redox_syscall 0.2.4", "remove_dir_all", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2666,7 +2531,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2700,28 +2565,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", - "fnv", - "futures-core", - "iovec", - "lazy_static", + "pin-project-lite 0.1.11", + "slab", +] + +[[package]] +name = "tokio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +dependencies = [ + "autocfg", + "bytes 1.0.1", "libc", "memchr", "mio", - "mio-named-pipes", - "mio-uds", "num_cpus", - "pin-project-lite 0.1.11", + "once_cell", + "parking_lot 0.11.1", + "pin-project-lite 0.2.4", "signal-hook-registry", - "slab", "tokio-macros", - "winapi 0.3.9", + "winapi", ] [[package]] name = "tokio-macros" -version = "0.2.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2", "quote", @@ -2730,74 +2602,66 @@ dependencies = [ [[package]] name = "tokio-native-tls" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd608593a919a8e05a7d1fc6df885e40f6a88d3a70a3a7eff23ff27964eda069" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio", + "tokio 1.2.0", ] [[package]] -name = "tokio-rustls" -version = "0.14.1" +name = "tokio-stream" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" dependencies = [ "futures-core", - "rustls", - "tokio", - "webpki", + "pin-project-lite 0.2.4", + "tokio 1.2.0", ] [[package]] name = "tokio-test" -version = "0.2.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed0049c119b6d505c4447f5c64873636c7af6c75ab0d45fd9f618d82acb8016d" +checksum = "7c7d205f6f59b03f9e824ac86eaba635a98395f287756ecc8a06464779c399bf" dependencies = [ - "bytes 0.5.6", + "async-stream", + "bytes 1.0.1", "futures-core", - "tokio", -] - -[[package]] -name = "tokio-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" -dependencies = [ - "native-tls", - "tokio", + "tokio 1.2.0", + "tokio-stream", ] [[package]] name = "tokio-tungstenite" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c" +checksum = "e1a5f475f1b9d077ea1017ecbc60890fda8e54942d680ca0b1d2b47cfa2d861b" dependencies = [ "futures-util", "log", "native-tls", - "pin-project 0.4.27", - "tokio", + "pin-project 1.0.5", + "tokio 1.2.0", "tokio-native-tls", "tungstenite", ] [[package]] name = "tokio-util" -version = "0.3.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "futures-core", + "futures-io", "futures-sink", "log", - "pin-project-lite 0.1.11", - "tokio", + "pin-project-lite 0.2.4", + "tokio 1.2.0", ] [[package]] @@ -2822,7 +2686,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d40a22fd029e33300d8d89a5cc8ffce18bb7c587662f54629e94c9de5487f3" dependencies = [ "cfg-if 1.0.0", - "log", "pin-project-lite 0.2.4", "tracing-attributes", "tracing-core", @@ -2909,19 +2772,19 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "tungstenite" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0308d80d86700c5878b9ef6321f020f29b1bb9d5ff3cab25e75e23f3a492a23" +checksum = "8ada8297e8d70872fa9a551d93250a9f407beb9f37ef86494eb20012a2ff7c24" dependencies = [ - "base64 0.12.3", + "base64", "byteorder", - "bytes 0.5.6", + "bytes 1.0.1", "http", "httparse", "input_buffer", "log", "native-tls", - "rand 0.7.3", + "rand 0.8.3", "sha-1", "url", "utf-8", @@ -2933,15 +2796,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.4" @@ -3141,25 +2995,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki-roots" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" -dependencies = [ - "webpki", -] - [[package]] name = "wepoll-sys" version = "3.0.1" @@ -3178,12 +3013,6 @@ dependencies = [ "libc", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -3194,12 +3023,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -3212,7 +3035,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3227,17 +3050,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] diff --git a/judger/Cargo.toml b/judger/Cargo.toml index e87ace0f..15749031 100644 --- a/judger/Cargo.toml +++ b/judger/Cargo.toml @@ -16,14 +16,14 @@ opt-level = "z" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] anyhow = "*" -arc-swap = "1.0" +arc-swap = "1.0.0" async-compat = "0.1" async-pipe = "0.1" async-tar = "0.3.0" async-trait = "0.1.42" -bollard = {path = "3rd_party/bollard"} +bollard = "0.10" broadcaster = {version = "1.0.0", features = ["default-channels"]} -bytes = "0.5.6" +bytes = "0.5" chrono = "0.4.19" clap = "3.0.0-beta.1" ctrlc = "3.1.7" @@ -33,9 +33,8 @@ dirs = "3.0.1" err-derive = "*" fern = "0.6.0" futures = "0.3.8" -futures_codec = "0.4.1" http = "*" -hyper = {version = "0.13.9", features = ["stream"]} +hyper = {version = "0.14", features = ["stream"]} libc = "0.2.81" log = "*" names = "0.11.0" @@ -44,29 +43,24 @@ path-absolutize = "3.0.6" path-slash = "0.1.3" rand = "0.7.3" regex = "1.4.2" -reqwest = {version = "0.10.9", features = ["json", "stream"]} +reqwest = {version = "0.11", features = ["json", "stream"]} rquickjs = {version = "0.1.1", features = ["bindgen", "parallel", "futures", "macro"]} serde = {version = "1.0.118", features = ["derive"]} serde_json = "1.0.60" shell-words = "1" tar = "0.4.30" -tokio = {version = "0.2.22", features = [ - "time", - "io-util", - "process", - "stream", - "rt-threaded", - "macros", +tokio = {version = "1", features = [ + "full", ]} -tokio-tungstenite = {version = "0.11.0", features = ["tls"]} -tokio-util = "0.3.1" +tokio-stream = {version = "0.1", features = ["fs"]} +tokio-tungstenite = {version = "0.13.0", features = ["tls"]} +tokio-util = {version = "0.6", features = ["codec", "compat"]} toml = "0.5.7" -tracing = "0.1.22" +tracing = "0.1.21" tracing-futures = "0.2.4" tracing-log = "0.1.1" tracing-subscriber = "0.2.15" -tungstenite = "0.11.1" [dev-dependencies] pretty_assertions = "0.6.1" -tokio-test = "0.2.1" +tokio-test = "0.4" diff --git a/judger/src/client.rs b/judger/src/client.rs index 742c8d7e..8f8813b0 100644 --- a/judger/src/client.rs +++ b/judger/src/client.rs @@ -21,10 +21,10 @@ use model::*; use serde_json::from_slice; use sink::*; use std::{collections::HashMap, path::PathBuf, sync::atomic::Ordering, sync::Arc}; -use tokio_tungstenite::{connect_async, tungstenite}; + +use tokio_tungstenite::{connect_async, tungstenite::Message}; use tracing::info_span; use tracing_futures::Instrument; -use tungstenite::Message; // Arc>==>>Arc @@ -260,7 +260,7 @@ pub async fn check_download_read_test_suite( pub async fn handle_job_wrapper( job: NewJob, send: Arc, - cancel: CancellationToken, + cancel: CancellationTokenHandle, cfg: Arc, ) { // TODO: Handle failed cases and report @@ -325,18 +325,23 @@ pub async fn handle_job_wrapper( flag_finished_job(send.clone(), cfg.clone()).await; - cfg.running_job_handles.lock().await.remove(&job_id); + tracing::info!("{}: Result message sent", job_id); + + { + cfg.running_job_handles.lock().await.remove(&job_id); + } match fs::ensure_removed_dir(&cfg.job_folder(job_id)).await { Ok(_) => {} Err(e) => tracing::error!("Failed to remove directory for job {}: {}", job_id, e), }; + tracing::info!("{}: cleanup complete", job_id); } pub async fn handle_job( job: NewJob, send: Arc, - cancel: CancellationToken, + cancel: CancellationTokenHandle, cfg: Arc, ) -> Result { let job = job.job; @@ -434,7 +439,7 @@ pub async fn handle_job( let ws_send = send.clone(); let job_id = job.id; async move { - while let Some((key, res)) = recv.next().await { + while let Some((key, res)) = recv.recv().await { tracing::info!("Job {}: recv message for key={}", job_id, key); // Omit error; it doesn't matter let _ = ws_send @@ -456,7 +461,7 @@ pub async fn handle_job( let ws_send = send.clone(); let job_id = job.id; async move { - while let Some(res) = recv.next().await { + while let Some(res) = recv.recv().await { let _ = ws_send .send_msg(&ClientMsg::JobOutput(JobOutputMsg { job_id, @@ -537,8 +542,8 @@ pub async fn flag_finished_job(send: Arc, client_config: Arc, client_config: Arc) { tracing::info!("Received job {}", job.job.id); let job_id = job.job.id; - let cancel_handle = client_config.cancel_handle.create_child(); - let cancel_token = cancel_handle.get_token(); + let cancel_handle = client_config.cancel_handle.child_token(); + let cancel_token = cancel_handle.child_token(); let handle = tokio::spawn(handle_job_wrapper( job, send, @@ -553,11 +558,13 @@ pub async fn accept_job(job: NewJob, send: Arc, client_config: Arc) { - let job = client_config - .running_job_handles - .lock() - .await - .remove(&job_id); + let job = { + client_config + .running_job_handles + .lock() + .await + .remove(&job_id) + }; if let Some((handle, cancel)) = job { cancel.cancel(); match handle.await { @@ -579,12 +586,15 @@ async fn keepalive( ws: Arc, interval: std::time::Duration, ) { - while tokio::time::delay_for(interval) - .with_cancel(client_config.cancel_handle.get_token()) + while tokio::time::sleep(interval) + .with_cancel(client_config.cancel_handle.child_token()) .await .is_some() { - match { ws.send_conf(tungstenite::Message::Ping(vec![]), true).await } { + match { + ws.send_conf(tokio_tungstenite::tungstenite::Message::Ping(vec![]), true) + .await + } { Ok(_) => {} Err(e) => { keepalive_token.cancel(); @@ -611,8 +621,8 @@ pub async fn client_loop( .await .unwrap(); - let keepalive_token = client_config.cancel_handle.create_child(); - let keepalive_cancel = keepalive_token.create_child(); + let keepalive_token = client_config.cancel_handle.child_token(); + let _keepalive_cancel = keepalive_token.child_token(); let keepalive_handle = tokio::spawn(keepalive( client_config.clone(), @@ -623,7 +633,7 @@ pub async fn client_loop( while let Some(Some(Ok(x))) = ws_recv .next() - .with_cancel(keepalive_cancel.get_token()) + .with_cancel(client_config.cancel_handle.child_token()) .await { let x: Message = x; diff --git a/judger/src/client/config.rs b/judger/src/client/config.rs index 8ccbf8f4..608f800c 100644 --- a/judger/src/client/config.rs +++ b/judger/src/client/config.rs @@ -1,4 +1,4 @@ -use crate::prelude::{CancellationToken, CancellationTokenHandle, FlowSnake}; +use crate::prelude::{CancellationTokenHandle, FlowSnake}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, path::PathBuf, sync::atomic::AtomicBool, sync::atomic::AtomicUsize, @@ -46,7 +46,7 @@ pub struct SharedClientData { /// HTTP client pub client: reqwest::Client, /// All test suites whose folder is being edited. - pub locked_test_suite: dashmap::DashMap, + pub locked_test_suite: dashmap::DashMap, /// Handle for all jobs currently running pub running_job_handles: Mutex, CancellationTokenHandle)>>, /// Handle for all jobs currently cancelling @@ -199,19 +199,20 @@ impl SharedClientData { } pub async fn obtain_suite_lock(&self, suite_id: FlowSnake) -> Option { + let state = rand::random(); let handle = CancellationTokenHandle::new(); let entry = self .locked_test_suite .entry(suite_id) - .or_insert_with(|| handle.get_token()) + .or_insert_with(|| (state, handle.child_token())) .clone(); tracing::debug!("Trying to obtain suite lock for {}", suite_id); - if entry.is_token_of(&handle) { + if entry.0 == state { tracing::debug!("Lock obtained"); - Some(handle) + Some(entry.1) } else { tracing::debug!("Already locked"); - entry.await; + (entry.1).cancelled().await; tracing::debug!("Lock cleared"); None } diff --git a/judger/src/client/sink.rs b/judger/src/client/sink.rs index 86aa5be8..c5525020 100644 --- a/judger/src/client/sink.rs +++ b/judger/src/client/sink.rs @@ -5,10 +5,10 @@ use arc_swap::{ArcSwapAny, ArcSwapOption}; use async_trait::async_trait; use futures::{ stream::{SplitSink, SplitStream}, - Sink, SinkExt, Stream, TryStream, + Sink, SinkExt, Stream, TryStream, }; use serde::Serialize; -use std::{ fmt::Debug, sync::Arc}; +use std::{fmt::Debug, sync::Arc}; use tokio::{net::TcpStream, sync::Mutex}; use tokio_tungstenite::{tungstenite, MaybeTlsStream, WebSocketStream}; use tungstenite::Message; @@ -50,8 +50,8 @@ impl WebsocketSink { // drop guard to avoid deadlock drop(sink); // wait for sink being connected - let handle = self.handle.load().create_child(); - handle.get_token().await; + let handle = self.handle.load().child_token(); + handle.cancelled().await; sink = self.sink.load(); } sink.clone().unwrap().lock().await.send(msg).await @@ -84,11 +84,15 @@ impl WebsocketSink { // drop guard to avoid deadlock drop(sink); // wait for sink being connected - let handle = self.handle.load().create_child(); - handle.get_token().await; + let handle = self.handle.load().child_token(); + handle.cancelled().await; sink = self.sink.load(); } - sink.clone().unwrap().lock().await.send_all(msg).await + let sink = sink.clone().unwrap(); + let mut sink = sink.lock().await; + let inner = &mut *sink; + inner.send_all(msg).await?; + Ok(()) } pub fn load_socket(&self, sink: RawWsSink) { diff --git a/judger/src/fs/mod.rs b/judger/src/fs/mod.rs index d877fe73..0d55044a 100644 --- a/judger/src/fs/mod.rs +++ b/judger/src/fs/mod.rs @@ -12,7 +12,7 @@ pub const JUDGE_FILE_NAME: &str = "judge.toml"; pub fn ensure_removed_dir(path: &Path) -> BoxFuture> { async move { let entries = match read_dir(path).await { - Ok(dir) => dir, + Ok(dir) => tokio_stream::wrappers::ReadDirStream::new(dir), Err(e) => match e.kind() { std::io::ErrorKind::NotFound => return Ok(()), _ => return Err(e), @@ -47,7 +47,7 @@ pub fn ensure_removed_dir(path: &Path) -> BoxFuture> pub fn find_judge_root(path: &Path) -> BoxFuture> { async move { - let mut dir = read_dir(path).await?; + let mut dir = tokio_stream::wrappers::ReadDirStream::new(read_dir(path).await?); let mut dirs = vec![]; let mut files = vec![]; while let Some(content) = dir.next().await { diff --git a/judger/src/fs/net.rs b/judger/src/fs/net.rs index e28c7be5..91983a3b 100644 --- a/judger/src/fs/net.rs +++ b/judger/src/fs/net.rs @@ -1,8 +1,8 @@ //! Functions to download stuff into destinations -use futures::{ StreamExt}; +use futures::StreamExt; use std::fmt::Write; use std::path::Path; -use tokio::io::{ AsyncWriteExt}; +use tokio::io::AsyncWriteExt; use tokio::process::Command; #[derive(Debug)] @@ -102,7 +102,7 @@ pub async fn download_unzip( log::info!("Writing {} bytes into {}", bytes.len(), dir.display()); file.write_all(&bytes).await?; } - file.flush(); + file.flush().await?; drop(file); let unzip_res = Command::new("7z") diff --git a/judger/src/main.rs b/judger/src/main.rs index fed5cdeb..f110aab1 100644 --- a/judger/src/main.rs +++ b/judger/src/main.rs @@ -13,6 +13,7 @@ use std::{ }; use std::{path::Path, process::exit}; use std::{sync::Arc, time::Duration}; +use tokio_tungstenite::tungstenite; use tracing_subscriber::FmtSubscriber; mod opt; @@ -36,8 +37,7 @@ fn main() { ctrlc::set_handler(handle_ctrl_c).expect("Failed to set termination handler!"); - let mut rt = tokio::runtime::Builder::new() - .threaded_scheduler() + let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .expect("Failed to initialize runtime"); @@ -160,7 +160,7 @@ async fn client(cmd: opt::ConnectSubCmd) { Err(e) => { // Exponential wait time tracing::warn!("Failed to connect: {}", e); - tokio::time::delay_for(wait_time).await; + tokio::time::sleep(wait_time).await; wait_time = std::cmp::min(wait_time.mul_f64(1.6), MAX_WAIT_TIME); continue; } diff --git a/judger/src/prelude/cancel_token.rs b/judger/src/prelude/cancel_token.rs index f9278ebc..d280764c 100644 --- a/judger/src/prelude/cancel_token.rs +++ b/judger/src/prelude/cancel_token.rs @@ -1,272 +1,11 @@ use async_trait::async_trait; -use dashmap::DashMap; use futures::{pin_mut, Future, FutureExt}; -use std::{ - num::NonZeroUsize, sync::atomic::AtomicBool, sync::atomic::AtomicUsize, sync::atomic::Ordering, - sync::Arc, sync::Weak, task::Poll, task::Waker, -}; -/// A handle for controlling cancellation tokens. -/// -/// This struct can be cloned to control the same tokens from different places. -/// -/// This implementation of `CancellationToken` is very inefficient, and should be -/// replaced with `tokio_util::sync::CancellationToken` once it is released -/// along `tokio_util` v0.4.0. -/// -/// # Structure -/// -/// ```plaintext -/// -> Waker 1 -> Waker 3 -/// |-> Waker 2 |-> Waker 4 -/// | | -/// * root_inner_token ----> * child_inner_token -/// | ^---------------------| -/// parent_token - - - - - > child_token -/// ``` -#[derive(Debug, Clone)] -pub struct CancellationTokenHandle { - token_ref: Option>, -} - -impl CancellationTokenHandle { - /// Generate a new handle for cancelling stuff - pub fn new() -> CancellationTokenHandle { - CancellationTokenHandle { - token_ref: Some(Arc::new(InnerCToken::new())), - } - } - - pub fn new_with_parent(parent: &CancellationTokenHandle) -> CancellationTokenHandle { - if let Some(parent) = parent.token_ref.clone() { - let token_ref = InnerCToken::new_with_parent(parent); - CancellationTokenHandle { - token_ref: Some(token_ref), - } - } else { - Default::default() - } - } - - /// Send a cancel signal for all tokens currently connected to this token - pub fn cancel(&self) { - if let Some(r) = self.token_ref.as_ref() { - r.wake_all(); - } - } - - pub fn is_cancelled(&self) -> bool { - self.token_ref - .as_ref() - .map(|r| r.is_cancelled()) - .unwrap_or(false) - } - - pub fn create_child(&self) -> CancellationTokenHandle { - Self::new_with_parent(self) - } - - /// Get a new token from this handle. - pub fn get_token(&self) -> CancellationToken { - CancellationToken { - token_ref: self.token_ref.clone(), - waker_id: None, - } - } - - /// Generate an empty handle that does nothing - pub fn empty() -> CancellationTokenHandle { - Self::default() - } - - pub fn is_empty(&self) -> bool { - self.token_ref.is_none() - } -} - -impl Default for CancellationTokenHandle { - /// Generate an empty handle that does nothing - fn default() -> Self { - CancellationTokenHandle { token_ref: None } - } -} - -impl Drop for CancellationTokenHandle { - fn drop(&mut self) { - if let Some(x) = self.token_ref.as_ref() { - if let Some((id, parent)) = x.parent.as_ref() { - parent.drop_child(*id); - } - } - } -} - -#[derive(Debug)] -struct InnerCToken { - cancelled: AtomicBool, - counter: AtomicUsize, - wakers: DashMap, - children: DashMap>, - parent: Option<(NonZeroUsize, Arc)>, -} - -impl InnerCToken { - pub fn new() -> Self { - InnerCToken { - cancelled: AtomicBool::new(false), - counter: AtomicUsize::new(1), - wakers: DashMap::new(), - children: DashMap::new(), - parent: None, - } - } - - pub fn new_with_parent(parent: Arc) -> Arc { - let this = Arc::new(Self::new()); - this.cancelled - .store(parent.cancelled.load(Ordering::SeqCst), Ordering::SeqCst); - let child_id = parent.store_child(&this); - let this_ptr = Arc::into_raw(this.clone()); - unsafe { - // * HI, UNSAFE! - // - // This code is safe because the inner value only has two references - // (`this` and `parent.children`). It's pretty much a custom - // `OnceCell` without all those clutter. - let this_ptr = this_ptr as *mut InnerCToken; - (*this_ptr).parent = Some((child_id, parent)); - let _ = Arc::from_raw(this_ptr); - } - this - } - - /// Store a waker reference generated by a context for waking up afterwards - pub fn store_waker(&self, waker: Waker) -> NonZeroUsize { - let id = NonZeroUsize::new(self.counter.fetch_add(1, Ordering::SeqCst)).unwrap(); - self.wakers.insert(id, waker); - id - } - - /// Drop the waker reference specified by this ID - pub fn drop_waker(&self, id: NonZeroUsize) -> Option { - self.wakers.remove(&id).map(|(_id, waker)| waker) - } - - /// Store a child reference generated by a context for waking up afterwards - pub fn store_child(&self, child: &Arc) -> NonZeroUsize { - let id = NonZeroUsize::new(self.counter.fetch_add(1, Ordering::SeqCst)).unwrap(); - self.children.insert(id, Arc::downgrade(child)); - id - } - - /// Drop the child reference specified by this ID - pub fn drop_child(&self, id: NonZeroUsize) { - self.children.remove(&id).map(|(_id, child)| child); - } - - /// Trigger all wakers and clean them up - pub fn wake_all(&self) { - self.cancelled.store(true, Ordering::Release); - self.wakers - .iter() - .for_each(|pair| pair.value().wake_by_ref()); - self.children.iter().for_each(|child| { - if let Some(x) = child.value().upgrade() { - x.wake_all() - } - }); - } - - pub fn is_cancelled(&self) -> bool { - self.cancelled.load(Ordering::Acquire) - } -} - -/// A cancellation token, also a future that can be awaited. -/// -/// This future resolves once the task is being cancelled. -#[derive(Debug)] -pub struct CancellationToken { - token_ref: Option>, - waker_id: Option, -} - -impl CancellationToken { - pub fn is_cancelled(&self) -> bool { - self.token_ref - .as_ref() - .map(|r| r.is_cancelled()) - .unwrap_or(false) - } - - pub fn is_token_of(&self, handle: &CancellationTokenHandle) -> bool { - handle.token_ref.as_ref().map_or(false, |r| { - self.token_ref.as_ref().map_or(false, |s| Arc::ptr_eq(r, s)) - }) - } -} - -impl Clone for CancellationToken { - /// Create a new cancellation token connected to the same handle instance. - /// - /// This method is essentially the same as `CancellationTokenHandle::get_token`, - /// just you don't need to go to the handle to get new tokens. - /// - /// This method is very cheap. - fn clone(&self) -> Self { - CancellationToken { - token_ref: self.token_ref.clone(), - waker_id: None, - } - } -} - -impl Future for CancellationToken { - type Output = (); - - fn poll( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> Poll { - if let Some(token_ref) = self.token_ref.clone() { - if token_ref.cancelled.load(Ordering::Acquire) { - if let Some(id) = self.waker_id.take() { - token_ref.drop_waker(id); - } - return Poll::Ready(()); - } - if let Some(_id) = self.waker_id.as_ref() { - // noop - } else { - let id = token_ref.store_waker(cx.waker().clone()); - self.waker_id = Some(id); - } - Poll::Pending - } else { - log::info!("eternity"); - Poll::Pending - } - } -} +// as tokio_util releases v0.4.0, these types are no longer required. -impl Drop for CancellationToken { - fn drop(&mut self) { - if let Some(token_ref) = self.token_ref.as_ref() { - if let Some(id) = self.waker_id.take() { - token_ref.drop_waker(id); - } - } - } -} +pub type CancellationTokenHandle = tokio_util::sync::CancellationToken; -impl Default for CancellationToken { - fn default() -> Self { - CancellationToken { - token_ref: None, - waker_id: None, - } - } -} +pub type CancellationToken<'a> = tokio_util::sync::WaitForCancellationFuture<'a>; #[async_trait] pub trait CancelFutureExt { @@ -274,9 +13,7 @@ pub trait CancelFutureExt { /// Execute this task with the given cancellation token, returning `None` /// if the task is being cancelled and `Some(output)` otherwise. - async fn with_cancel(self, mut cancel: C) -> Option - where - C: ICancellationToken; + async fn with_cancel(self, mut cancel: CancellationTokenHandle) -> Option; } #[async_trait] @@ -286,81 +23,13 @@ where { type Output = T::Output; - async fn with_cancel(self, cancel: C) -> Option - where - C: ICancellationToken, - { + async fn with_cancel(self, cancel: CancellationTokenHandle) -> Option { let self_ = self.fuse(); pin_mut!(self_); - futures::select! { - _abort = cancel.fuse() => None, - fut = self_ => Some(fut), - complete => None + tokio::select! { + _abort = cancel.cancelled() => None, + fut = self_ => Some(fut) } } } - -pub trait ICancellationToken: Future + Send + Unpin {} - -impl ICancellationToken for CancellationToken {} - -#[cfg(test)] -mod test { - use super::*; - use std::time::Duration; - - #[test] - fn cancel_token_should_not_be_triggered() { - let handle = CancellationTokenHandle::new(); - let res = tokio_test::block_on(async move { - let token = handle.get_token(); - let awaiter = tokio::time::delay_for(Duration::from_secs(5)); - awaiter.with_cancel(token).await - }); - assert_eq!(res, Some(())) - } - - #[test] - fn cancel_token_being_triggered() { - let handle = CancellationTokenHandle::new(); - let res = tokio_test::block_on(async move { - let token = handle.get_token(); - let awaiter = tokio::time::delay_for(Duration::from_secs(3600)); - futures::join!(awaiter.with_cancel(token), async { handle.cancel() }) - }); - assert_eq!(res, (None, ())) - } - - #[test] - fn multiple_cancel_token_being_triggered() { - let handle = CancellationTokenHandle::new(); - let res = tokio_test::block_on(async move { - let token = handle.get_token(); - let token2 = handle.get_token(); - let awaiter = tokio::time::delay_for(Duration::from_secs(3600)); - let awaiter2 = tokio::time::delay_for(Duration::from_secs(3600)); - futures::join!( - awaiter.with_cancel(token), - awaiter2.with_cancel(token2), - async { handle.cancel() } - ) - }); - assert_eq!(res, (None, None, ())) - } - - #[test] - fn child_token_being_triggered() { - let handle = CancellationTokenHandle::new(); - let res = tokio_test::block_on(async move { - let child_handle = handle.create_child(); - let token = child_handle.get_token(); - let awaiter = tokio::time::delay_for(Duration::from_secs(3600)); - futures::join!(awaiter.with_cancel(token), async move { - let child = child_handle; - handle.cancel() - }) - }); - assert_eq!(res, (None, ())) - } -} diff --git a/judger/src/tester/exec.rs b/judger/src/tester/exec.rs index 4db3098e..3b860857 100644 --- a/judger/src/tester/exec.rs +++ b/judger/src/tester/exec.rs @@ -14,7 +14,6 @@ use crate::{ prelude::*, }; use anyhow::Result; -use async_compat::CompatExt; use bollard::models::{BuildInfo, Mount}; use futures::stream::StreamExt; use once_cell::sync::Lazy; @@ -22,8 +21,9 @@ use path_slash::PathBufExt; use std::path::Path; use std::time; use std::{collections::HashMap, io, path::PathBuf, string::String, sync::Arc}; -use tokio::io::{AsyncReadExt, BufWriter}; +use tokio::io::{AsyncReadExt}; use tokio::sync::mpsc::UnboundedSender; +use tokio_util::compat::*; #[cfg(unix)] use super::utils::strsignal; @@ -334,7 +334,7 @@ impl Image { &self, instance: bollard::Docker, partial_result_channel: Option, - cancel: CancellationToken, + cancel: CancellationTokenHandle, ) -> Result<(), BuildError> { match &self { Image::Image { tag } => { @@ -364,11 +364,13 @@ impl Image { } Image::Dockerfile { tag, path, file } => { let from_path = path.clone(); - let (pipe_recv, pipe_send) = async_pipe::pipe(); + let (pipe_recv, pipe_send) = tokio::io::duplex(8192); let read_codec = tokio_util::codec::BytesCodec::new(); let frame = tokio_util::codec::FramedRead::new(pipe_send, read_codec); let task = async move { - let mut tar = async_tar::Builder::new(BufWriter::new(pipe_recv).compat()); + let mut tar = async_tar::Builder::new(futures::io::BufWriter::new( + pipe_recv.compat_write(), + )); match tar.append_dir_all(".", from_path).await { Ok(_) => tar.finish().await, e @ Err(_) => e, @@ -593,7 +595,7 @@ impl TestSuite { build_result_channel: Option, result_channel: Option>, upload_info: Option>, - cancellation_token: CancellationToken, + cancellation_token: CancellationTokenHandle, ) -> anyhow::Result> { let rnd_id = rand::random::(); let TestSuiteOptions { @@ -801,560 +803,5 @@ fn construct_case_index(pub_cfg: &JudgerPublicConfig) -> HashMap = Err(JobFailure::ExecError(ExecError { - stage: 1, - kind: ExecErrorKind::ReturnCodeCheckFailed, - output: vec![ - ProcessInfo { - ret_code: 0, - command: "echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - is_user_command: true, - }, - ProcessInfo { - ret_code: 1, - command: "echo 'Hello, world!' && false".into(), - stdout: "Hello, world!\n".into(), - stderr: "".into(), - is_user_command: true, - }, - ], - })); - pretty_eq!(got, expected); - }) - } - - #[test] - fn signal() { - block_on(async { - let mut t = Test::new(); - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step(Step::new(Capturable::new( - // Kill a running task - r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into() - ),true)); - t.expected("Hello,\nworld!\n"); - let got = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; - let expected: Result = Err(JobFailure::ExecError(ExecError { - stage: 1, - kind: ExecErrorKind::RuntimeError( - format!( - "Runtime Error: {}", - strsignal(15) - ) - ), - output: vec![ - ProcessInfo { - ret_code: 0, - is_user_command:true, - command: r"echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - }, - ProcessInfo { - ret_code: -15, - is_user_command:true, - command:r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into(), - stdout: "0\n".into(), - stderr: "".into(), - }, - ], - })); - pretty_eq!(got, expected); - }) - } - - #[test] - fn output_mismatch() { - block_on(async { - let mut t = Test::new(); - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step(Step::new( - Capturable::new("echo 'Hello, world!' | awk '{print $2}'".into()), - true, - )); - t.expected("Hello,\nworld!"); - let got = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; - let expected: Result = Err(JobFailure::OutputMismatch(OutputMismatch { - diff: "+ Hello,\n world!\n".into(), - output: vec![ - ProcessInfo { - ret_code: 0, - is_user_command: true, - command: r"echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - }, - ProcessInfo { - ret_code: 0, - is_user_command: true, - command: "echo 'Hello, world!' | awk '{print $2}'".into(), - stdout: "world!\n".into(), - stderr: "".into(), - }, - ], - })); - pretty_eq!(got, expected); - }) - } - - #[test] - fn output_timed_out() { - block_on(async { - let mut t = Test::new(); - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step( - Step::new(Capturable::new("echo 0; sleep 3; echo 1".into()), true) - .timeout(time::Duration::from_millis(100)), - ); - t.expected("Hello,\nworld!\n"); - let got = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; - let expected: Result = Err(JobFailure::ExecError(ExecError { - stage: 1, - kind: ExecErrorKind::TimedOut, - output: vec![ProcessInfo { - ret_code: 0, - is_user_command: true, - command: r"echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - }], - })); - pretty_eq!(got, expected); - }) - } - } - - mod docker_runner { - use super::*; - use crate::tester::runner::{DockerCommandRunner, DockerCommandRunnerOptions}; - - fn docker_run(f: F) - where - F: FnOnce(DockerCommandRunner, Test) -> O, - O: futures::Future, - { - block_on(async { - let runner = DockerCommandRunner::try_new( - bollard::Docker::connect_with_local_defaults().unwrap(), - Image::Image { - tag: "alpine:latest".to_owned(), - }, - DockerCommandRunnerOptions { - build_image: true, - ..Default::default() - }, - Option::::None, - ) - .await - .unwrap(); - let t = Test::new(); - f(runner, t).await.kill().await; - }); - } - - #[test] - fn ok() { - docker_run(|runner, mut t| async { - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step(Step::new( - Capturable::new("echo 'Hello, world!' | awk '{print $1}'".into()), - true, - )); - t.expected("Hello,\n"); - let res = t.run(&runner, &HashMap::new(), None).await; - // Any Ok(_) represents accepted, just with different score. - assert!(matches!(dbg!(res), Ok(_))); - runner - }); - } - - #[test] - fn error_code() { - docker_run(|runner, mut t| async { - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step(Step::new( - Capturable::new("echo 'Hello, world!' && false".into()), - true, - )); - t.expected("Hello,\nworld!\n"); - let got = t.run(&runner, &HashMap::new(), None).await; - let expected: Result = Err(JobFailure::ExecError(ExecError { - stage: 1, - kind: ExecErrorKind::ReturnCodeCheckFailed, - output: vec![ - ProcessInfo { - ret_code: 0, - command: "echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - is_user_command: true, - }, - ProcessInfo { - ret_code: 1, - command: "echo 'Hello, world!' && false".into(), - stdout: "Hello, world!\n".into(), - stderr: "".into(), - is_user_command: true, - }, - ], - })); - pretty_eq!(got, expected); - runner - }) - } - - #[test] - fn signal() { - docker_run(|runner, mut t| async { - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step(Step::new(Capturable::new( - // Kill a running task - r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into() - ),true)); - t.expected("Hello,\nworld!\n"); - let got = t.run(&runner, &HashMap::new(), None).await; - let expected: Result = Err(JobFailure::ExecError(ExecError { - stage: 1, - kind: if cfg!(unix){ ExecErrorKind::RuntimeError( - format!( - "Runtime Error: {}", - strsignal(15) - ) - )}else{ - ExecErrorKind::ReturnCodeCheckFailed - }, - output: vec![ - ProcessInfo { - ret_code: 0, - is_user_command:true, - command: r"echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - }, - ProcessInfo { - ret_code: -15, - is_user_command:true, - command:r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into(), - stdout: "0\n".into(), - stderr: "".into(), - }, - ], - })); - pretty_eq!(got, expected); - runner - }) - } - - #[test] - fn output_mismatch() { - docker_run(|runner, mut t| async { - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step(Step::new( - Capturable::new("echo 'Hello, world!' | awk '{print $2}'".into()), - true, - )); - t.expected("Hello,\nworld!"); - let got = t.run(&runner, &HashMap::new(), None).await; - let expected: Result = Err(JobFailure::OutputMismatch(OutputMismatch { - diff: "+ Hello,\n world!\n".into(), - output: vec![ - ProcessInfo { - ret_code: 0, - is_user_command: true, - command: r"echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - }, - ProcessInfo { - ret_code: 0, - is_user_command: true, - command: "echo 'Hello, world!' | awk '{print $2}'".into(), - stdout: "world!\n".into(), - stderr: "".into(), - }, - ], - })); - pretty_eq!(got, expected); - runner - }) - } - - #[test] - fn output_timed_out() { - docker_run(|runner, mut t| async { - t.add_step(Step::new( - Capturable::new(r"echo 'This does nothing.'".into()), - true, - )); - t.add_step( - Step::new(Capturable::new("echo 0; sleep 3; echo 1".into()), true) - .timeout(time::Duration::from_millis(100)), - ); - t.expected("Hello,\nworld!\n"); - let got = t.run(&runner, &HashMap::new(), None).await; - let expected: Result = Err(JobFailure::ExecError(ExecError { - stage: 1, - kind: ExecErrorKind::TimedOut, - output: vec![ProcessInfo { - ret_code: 0, - is_user_command: true, - command: r"echo 'This does nothing.'".into(), - stdout: "This does nothing.\n".into(), - stderr: "".into(), - }], - })); - pretty_eq!(got, expected); - runner - }) - } - } -} - -#[cfg(test)] -mod test_suite { - use super::*; - use tokio_test::block_on; - - #[test] - fn golem_no_volume() -> Result<()> { - block_on(async { - let image_name = "golem_no_volume"; - // Repo directory in the host FS. - let host_repo_dir = PathBuf::from(r"../golem"); - - let mut ts = TestSuite::from_config( - Image::Dockerfile { - tag: image_name.to_owned(), - path: host_repo_dir, - file: None, - }, - &std::env::current_dir().unwrap(), - JudgerPrivateConfig { - test_root_dir: PathBuf::from(r"../golem/src"), - mapped_test_root_dir: PathBuf::from(r"/golem/src"), - }, - JudgerPublicConfig { - time_limit: None, - memory_limit: None, - name: "golem_no_volume".into(), - test_groups: { - [( - "default".to_owned(), - vec![TestCaseDefinition { - name: "succ".into(), - should_fail: false, - has_out: true, - base_score: 1.0, - }], - )] - .iter() - .cloned() - .collect() - }, - vars: [ - ("$src", "py"), - ("$bin", "pyc"), - ("$stdin", "in"), - ("$stdout", "out"), - ] - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - run: ["cat $stdin | python ./golem.py $bin"] - .iter() - .map(|s| s.to_string()) - .collect(), - - mapped_dir: Bind { - from: PathBuf::from(r"../golem/src"), - to: PathBuf::from(r"../golem/src"), - readonly: false, - }, - binds: None, - special_judge_script: None, - }, - &JudgeTomlTestConfig { - // TODO: Refine interface - image: Image::Image { tag: "".into() }, - build: None, - run: vec!["python ./golemc.py $src -o $bin".into()], - }, - TestSuiteOptions { - tests: ["succ"].iter().map(|s| s.to_string()).collect(), - time_limit: None, - mem_limit: None, - build_image: true, - remove_image: true, - }, - ) - .await?; - - let instance = bollard::Docker::connect_with_local_defaults().unwrap(); - ts.run( - instance, - std::env::current_dir().unwrap(), - None, - None, - None, - Default::default(), - ) - .await?; - Ok(()) - }) - } - - #[test] - fn golem_with_volume() -> Result<()> { - block_on(async { - let image_name = "golem"; - // Repo directory in the host FS. - let host_repo_dir = PathBuf::from(r"../golem"); - - let mut ts = TestSuite::from_config( - Image::Dockerfile { - tag: image_name.to_owned(), - path: host_repo_dir, // public: c# gives repo remote, rust clone and unzip - file: None, - }, - &std::env::current_dir().unwrap(), - JudgerPrivateConfig { - test_root_dir: PathBuf::from(r"../golem/src"), - mapped_test_root_dir: PathBuf::from(r"/golem/src"), - }, - JudgerPublicConfig { - time_limit: None, - memory_limit: None, - name: "golem".into(), - test_groups: { - [( - "default".to_owned(), - vec![TestCaseDefinition { - name: "succ".into(), - should_fail: false, - has_out: true, - base_score: 1.0, - }], - )] - .iter() - .cloned() - .collect() - }, - vars: [ - ("$src", "py"), - ("$bin", "pyc"), - ("$stdin", "in"), - ("$stdout", "out"), - ] // public - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - run: ["cat $stdin | python ./golem.py $bin"] // public - .iter() - .map(|s| s.to_string()) - .collect(), - - mapped_dir: Bind { - from: PathBuf::from(r"../golem/src"), - to: PathBuf::from(r"/golem/src"), - readonly: false, - }, - binds: Some(vec![]), - special_judge_script: None, - }, - &JudgeTomlTestConfig { - // TODO: Refine interface - image: Image::Image { tag: "".into() }, - build: None, - run: vec!["python ./golemc.py $src -o $bin".into()], - }, - TestSuiteOptions { - tests: ["succ"].iter().map(|s| s.to_string()).collect(), // private - time_limit: None, // private - mem_limit: None, // private - build_image: true, // private - remove_image: true, // private - }, - ) - .await?; - - let instance = bollard::Docker::connect_with_local_defaults().unwrap(); - ts.run( - instance, - std::env::current_dir().unwrap(), - None, - None, - None, - Default::default(), - ) - .await?; - Ok(()) - }) - } -} +mod tests; +mod test_suite; diff --git a/judger/src/tester/exec/test_suite.rs b/judger/src/tester/exec/test_suite.rs new file mode 100644 index 00000000..3ee5258f --- /dev/null +++ b/judger/src/tester/exec/test_suite.rs @@ -0,0 +1,179 @@ +#![cfg(test)] +use super::*; +use tokio_test::block_on; + +#[test] +fn golem_no_volume() -> Result<()> { + block_on(async { + let image_name = "golem_no_volume"; + // Repo directory in the host FS. + let host_repo_dir = PathBuf::from(r"../golem"); + + let mut ts = TestSuite::from_config( + Image::Dockerfile { + tag: image_name.to_owned(), + path: host_repo_dir, + file: None, + }, + &std::env::current_dir().unwrap(), + JudgerPrivateConfig { + test_root_dir: PathBuf::from(r"../golem/src"), + mapped_test_root_dir: PathBuf::from(r"/golem/src"), + }, + JudgerPublicConfig { + time_limit: None, + memory_limit: None, + name: "golem_no_volume".into(), + test_groups: { + [( + "default".to_owned(), + vec![TestCaseDefinition { + name: "succ".into(), + should_fail: false, + has_out: true, + base_score: 1.0, + }], + )] + .iter() + .cloned() + .collect() + }, + vars: [ + ("$src", "py"), + ("$bin", "pyc"), + ("$stdin", "in"), + ("$stdout", "out"), + ] + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + run: ["cat $stdin | python ./golem.py $bin"] + .iter() + .map(|s| s.to_string()) + .collect(), + + mapped_dir: Bind { + from: PathBuf::from(r"../golem/src"), + to: PathBuf::from(r"../golem/src"), + readonly: false, + }, + binds: None, + special_judge_script: None, + }, + &JudgeTomlTestConfig { + // TODO: Refine interface + image: Image::Image { tag: "".into() }, + build: None, + run: vec!["python ./golemc.py $src -o $bin".into()], + }, + TestSuiteOptions { + tests: ["succ"].iter().map(|s| s.to_string()).collect(), + time_limit: None, + mem_limit: None, + build_image: true, + remove_image: true, + }, + ) + .await?; + + let instance = bollard::Docker::connect_with_local_defaults().unwrap(); + ts.run( + instance, + std::env::current_dir().unwrap(), + None, + None, + None, + Default::default(), + ) + .await?; + Ok(()) + }) +} + +#[test] +fn golem_with_volume() -> Result<()> { + block_on(async { + let image_name = "golem"; + // Repo directory in the host FS. + let host_repo_dir = PathBuf::from(r"../golem"); + + let mut ts = TestSuite::from_config( + Image::Dockerfile { + tag: image_name.to_owned(), + path: host_repo_dir, // public: c# gives repo remote, rust clone and unzip + file: None, + }, + &std::env::current_dir().unwrap(), + JudgerPrivateConfig { + test_root_dir: PathBuf::from(r"../golem/src"), + mapped_test_root_dir: PathBuf::from(r"/golem/src"), + }, + JudgerPublicConfig { + time_limit: None, + memory_limit: None, + name: "golem".into(), + test_groups: { + [( + "default".to_owned(), + vec![TestCaseDefinition { + name: "succ".into(), + should_fail: false, + has_out: true, + base_score: 1.0, + }], + )] + .iter() + .cloned() + .collect() + }, + vars: [ + ("$src", "py"), + ("$bin", "pyc"), + ("$stdin", "in"), + ("$stdout", "out"), + ] // public + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + run: ["cat $stdin | python ./golem.py $bin"] // public + .iter() + .map(|s| s.to_string()) + .collect(), + + mapped_dir: Bind { + from: PathBuf::from(r"../golem/src"), + to: PathBuf::from(r"/golem/src"), + readonly: false, + }, + binds: Some(vec![]), + special_judge_script: None, + }, + &JudgeTomlTestConfig { + // TODO: Refine interface + image: Image::Image { tag: "".into() }, + build: None, + run: vec!["python ./golemc.py $src -o $bin".into()], + }, + TestSuiteOptions { + tests: ["succ"].iter().map(|s| s.to_string()).collect(), // private + time_limit: None, // private + mem_limit: None, // private + build_image: true, // private + remove_image: true, // private + }, + ) + .await?; + + let instance = bollard::Docker::connect_with_local_defaults().unwrap(); + ts.run( + instance, + std::env::current_dir().unwrap(), + None, + None, + None, + Default::default(), + ) + .await?; + Ok(()) + }) +} diff --git a/judger/src/tester/exec/tests.rs b/judger/src/tester/exec/tests.rs new file mode 100644 index 00000000..1f5bbcf0 --- /dev/null +++ b/judger/src/tester/exec/tests.rs @@ -0,0 +1,372 @@ +#![cfg(test)] +use super::*; +use pretty_assertions::assert_eq as pretty_eq; +use tokio_test::block_on; + +#[cfg(unix)] +mod tokio_runner { + use super::*; + use crate::tester::runner::TokioCommandRunner; + + #[test] + fn ok() { + block_on(async { + let mut t = Test::new(); + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new( + Capturable::new("echo 'Hello, world!' | awk '{print $1}'".into()), + true, + )); + t.expected("Hello,\n"); + let res = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; + assert!(matches!(dbg!(res), Ok(_))); + }) + } + + #[test] + fn error_code() { + block_on(async { + let mut t = Test::new(); + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new( + Capturable::new("echo 'Hello, world!' && false".into()), + true, + )); + t.expected("Goodbye, world!"); + let got = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::ExecError(ExecError { + stage: 1, + kind: ExecErrorKind::ReturnCodeCheckFailed, + output: vec![ + ProcessInfo { + ret_code: 0, + command: "echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + is_user_command: true, + }, + ProcessInfo { + ret_code: 1, + command: "echo 'Hello, world!' && false".into(), + stdout: "Hello, world!\n".into(), + stderr: "".into(), + is_user_command: true, + }, + ], + })); + pretty_eq!(got, expected); + }) + } + + #[test] + fn signal() { + block_on(async { + let mut t = Test::new(); + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new(Capturable::new( + // Kill a running task + r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into() + ),true)); + t.expected("Hello,\nworld!\n"); + let got = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::ExecError(ExecError { + stage: 1, + kind: ExecErrorKind::RuntimeError( + format!( + "Runtime Error: {}", + strsignal(15) + ) + ), + output: vec![ + ProcessInfo { + ret_code: 0, + is_user_command:true, + command: r"echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + }, + ProcessInfo { + ret_code: -15, + is_user_command:true, + command:r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into(), + stdout: "0\n".into(), + stderr: "".into(), + }, + ], + })); + pretty_eq!(got, expected); + }) + } + + #[test] + fn output_mismatch() { + block_on(async { + let mut t = Test::new(); + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new( + Capturable::new("echo 'Hello, world!' | awk '{print $2}'".into()), + true, + )); + t.expected("Hello,\nworld!"); + let got = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::OutputMismatch(OutputMismatch { + diff: "+ Hello,\n world!\n".into(), + output: vec![ + ProcessInfo { + ret_code: 0, + is_user_command: true, + command: r"echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + }, + ProcessInfo { + ret_code: 0, + is_user_command: true, + command: "echo 'Hello, world!' | awk '{print $2}'".into(), + stdout: "world!\n".into(), + stderr: "".into(), + }, + ], + })); + pretty_eq!(got, expected); + }) + } + + #[test] + fn output_timed_out() { + block_on(async { + let mut t = Test::new(); + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step( + Step::new(Capturable::new("echo 0; sleep 3; echo 1".into()), true) + .timeout(time::Duration::from_millis(100)), + ); + t.expected("Hello,\nworld!\n"); + let got = t.run(&TokioCommandRunner {}, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::ExecError(ExecError { + stage: 1, + kind: ExecErrorKind::TimedOut, + output: vec![ProcessInfo { + ret_code: 0, + is_user_command: true, + command: r"echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + }], + })); + pretty_eq!(got, expected); + }) + } +} + +mod docker_runner { + use super::*; + use crate::tester::runner::{DockerCommandRunner, DockerCommandRunnerOptions}; + + fn docker_run(f: F) + where + F: FnOnce(DockerCommandRunner, Test) -> O, + O: futures::Future, + { + block_on(async { + let runner = DockerCommandRunner::try_new( + bollard::Docker::connect_with_local_defaults().unwrap(), + Image::Image { + tag: "alpine:latest".to_owned(), + }, + DockerCommandRunnerOptions { + build_image: true, + ..Default::default() + }, + Option::::None, + ) + .await + .unwrap(); + let t = Test::new(); + f(runner, t).await.kill().await; + }); + } + + #[test] + fn ok() { + docker_run(|runner, mut t| async { + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new( + Capturable::new("echo 'Hello, world!' | awk '{print $1}'".into()), + true, + )); + t.expected("Hello,\n"); + let res = t.run(&runner, &HashMap::new(), None).await; + // Any Ok(_) represents accepted, just with different score. + assert!(matches!(dbg!(res), Ok(_))); + runner + }); + } + + #[test] + fn error_code() { + docker_run(|runner, mut t| async { + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new( + Capturable::new("echo 'Hello, world!' && false".into()), + true, + )); + t.expected("Hello,\nworld!\n"); + let got = t.run(&runner, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::ExecError(ExecError { + stage: 1, + kind: ExecErrorKind::ReturnCodeCheckFailed, + output: vec![ + ProcessInfo { + ret_code: 0, + command: "echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + is_user_command: true, + }, + ProcessInfo { + ret_code: 1, + command: "echo 'Hello, world!' && false".into(), + stdout: "Hello, world!\n".into(), + stderr: "".into(), + is_user_command: true, + }, + ], + })); + pretty_eq!(got, expected); + runner + }) + } + + #[test] + fn signal() { + docker_run(|runner, mut t| async { + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new(Capturable::new( + // Kill a running task + r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into() + ),true)); + t.expected("Hello,\nworld!\n"); + let got = t.run(&runner, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::ExecError(ExecError { + stage: 1, + kind: if cfg!(unix){ ExecErrorKind::RuntimeError( + format!( + "Runtime Error: {}", + strsignal(15) + ) + )}else{ + ExecErrorKind::ReturnCodeCheckFailed + }, + output: vec![ + ProcessInfo { + ret_code: 0, + is_user_command:true, + command: r"echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + }, + ProcessInfo { + ret_code: -15, + is_user_command:true, + command:r#"{ sleep 0.1; kill $$; } & i=0; while [ "$i" -lt 4 ]; do echo $i; sleep 1; i=$(( i + 1 )); done"#.into(), + stdout: "0\n".into(), + stderr: "".into(), + }, + ], + })); + pretty_eq!(got, expected); + runner + }) + } + + #[test] + fn output_mismatch() { + docker_run(|runner, mut t| async { + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step(Step::new( + Capturable::new("echo 'Hello, world!' | awk '{print $2}'".into()), + true, + )); + t.expected("Hello,\nworld!"); + let got = t.run(&runner, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::OutputMismatch(OutputMismatch { + diff: "+ Hello,\n world!\n".into(), + output: vec![ + ProcessInfo { + ret_code: 0, + is_user_command: true, + command: r"echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + }, + ProcessInfo { + ret_code: 0, + is_user_command: true, + command: "echo 'Hello, world!' | awk '{print $2}'".into(), + stdout: "world!\n".into(), + stderr: "".into(), + }, + ], + })); + pretty_eq!(got, expected); + runner + }) + } + + #[test] + fn output_timed_out() { + docker_run(|runner, mut t| async { + t.add_step(Step::new( + Capturable::new(r"echo 'This does nothing.'".into()), + true, + )); + t.add_step( + Step::new(Capturable::new("echo 0; sleep 3; echo 1".into()), true) + .timeout(time::Duration::from_millis(100)), + ); + t.expected("Hello,\nworld!\n"); + let got = t.run(&runner, &HashMap::new(), None).await; + let expected: Result = Err(JobFailure::ExecError(ExecError { + stage: 1, + kind: ExecErrorKind::TimedOut, + output: vec![ProcessInfo { + ret_code: 0, + is_user_command: true, + command: r"echo 'This does nothing.'".into(), + stdout: "This does nothing.\n".into(), + stderr: "".into(), + }], + })); + pretty_eq!(got, expected); + runner + }) + } +} diff --git a/judger/src/tester/runner.rs b/judger/src/tester/runner.rs index b7e15910..673912c3 100644 --- a/judger/src/tester/runner.rs +++ b/judger/src/tester/runner.rs @@ -4,7 +4,6 @@ use super::utils::convert_code; use super::{JobFailure, ProcessInfo}; use crate::prelude::*; use anyhow::Result; -use async_compat::CompatExt; use async_trait::async_trait; use bollard::{container::UploadToContainerOptions, exec::StartExecResults, models::Mount, Docker}; use futures::stream::StreamExt; @@ -13,7 +12,8 @@ use names::{Generator, Name}; use std::os::unix::process::ExitStatusExt; use std::process::ExitStatus; use std::{collections::HashMap, default::Default}; -use tokio::{io::BufWriter, process::Command}; +use tokio::process::Command; +use tokio_util::compat::*; /// An evaluation environment for commands. #[async_trait] @@ -107,7 +107,7 @@ pub struct DockerCommandRunnerOptions { /// Data to be copied into container before build, in format of `(source_dir, target_dir)` pub copies: Option>, /// Token to cancel this runner - pub cancellation_token: CancellationToken, + pub cancellation_token: CancellationTokenHandle, } impl Default for DockerCommandRunnerOptions { @@ -235,11 +235,13 @@ impl DockerCommandRunner { .collect::, bollard::errors::Error>>()?; let from_path = from_path.clone(); - let (pipe_recv, pipe_send) = async_pipe::pipe(); + let (pipe_recv, pipe_send) = tokio::io::duplex(8192); let read_codec = tokio_util::codec::BytesCodec::new(); let frame = tokio_util::codec::FramedRead::new(pipe_send, read_codec); let task = async move { - let mut tar = async_tar::Builder::new(BufWriter::new(pipe_recv).compat()); + let mut tar = async_tar::Builder::new(futures::io::BufWriter::new( + pipe_recv.compat_write(), + )); match tar.append_dir_all(".", from_path).await { Ok(_) => tar.finish().await, e @ Err(_) => e, @@ -253,7 +255,7 @@ impl DockerCommandRunner { path: to_path.clone(), ..Default::default() }), - hyper::Body::wrap_stream(frame.map(|x| x.map(bytes::BytesMut::freeze))), + hyper::Body::wrap_stream(frame.map(|x| x)), ) .await?; task.await??;