diff --git a/Cargo.lock b/Cargo.lock index 4ae50c55..508fab7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.7.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb9843d84c775696c37d9a418bbb01b932629d01870722c0f13eb3f95e2536d" +checksum = "d48f96fc3003717aeb9856ca3d02a8c7de502667ad76eeacd830b48d2e91fac4" dependencies = [ "actix-codec", "actix-rt", @@ -81,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -101,9 +101,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" +checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" dependencies = [ "futures-core", "tokio", @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4" +checksum = "7ca2549781d8dd6d75c40cf6b6051260a2cc2f3c62343d761a969a0640646894" dependencies = [ "actix-rt", "actix-service", @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.6.0" +version = "4.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cf67dadb19d7c95e5a299e2dda24193b89d5d4f33a3b9800888ede9e19aa32" +checksum = "9180d76e5cc7ccbc4d60a506f2c727730b154010262df5b910eb17dbe4b8cb38" dependencies = [ "actix-codec", "actix-http", @@ -191,6 +191,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", + "impl-more", "itoa", "language-tags", "log", @@ -210,30 +211,30 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "4.2.2" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" +checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] name = "addr2line" -version = "0.21.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aead" @@ -322,9 +323,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", @@ -337,33 +338,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -371,9 +372,9 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "assert-json-diff" @@ -388,9 +389,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c90a406b4495d129f00461241616194cb8a032c8d1c53c657f0961d5f8e0498" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "flate2", "futures-core", @@ -418,18 +419,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -464,9 +465,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" -version = "1.4.0" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ddbfb5db93d62521f47b3f223da0884a2f02741ff54cb9cda192a0e73ba08b" +checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -484,7 +485,6 @@ dependencies = [ "fastrand", "hex", "http 0.2.12", - "hyper 0.14.28", "ring", "time", "tokio", @@ -495,9 +495,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -507,9 +507,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.7.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8487b59d62764df8231cb371c459314df895b41756df457a1fb1243d65c89195" +checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.16.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15eb61145320320eb919d9bab524617a7aa4216c78d342fae3a758bc33073e4" +checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62" dependencies = [ "bindgen", "cc", @@ -534,15 +534,16 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.2.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75588e7ee5e8496eed939adac2035a6dbab9f7eb2acdd9ab2d31856dab6f3955" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -550,6 +551,7 @@ dependencies = [ "fastrand", "http 0.2.12", "http-body 0.4.6", + "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -558,9 +560,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.29.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "966646a69665bb0427460d78747204317f6639bdf5ec61305c4c5195af3dc086" +checksum = "c09fd4b5c7ed75f52b913b4f3ff0501dae7f8cb9125f6d45db4553980cbc0528" dependencies = [ "ahash", "aws-credential-types", @@ -593,9 +595,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.25.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef2d9ca2b43051224ed326ed9960a85e277b7d554a2cd0397e57c0553d86e64" +checksum = "70a9d27ed1c12b1140c47daf1bc541606c43fdafd918c4797d520db0043ceef2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -615,9 +617,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.26.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c869d1f5c4ee7437b79c3c1664ddbf7a60231e893960cf82b2b299a5ccf2cc5d" +checksum = "44514a6ca967686cde1e2a1b81df6ef1883d0e3e570da8d8bc5c491dcb6fc29b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -637,9 +639,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.25.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2b4a632a59e4fab7abf1db0d94a3136ad7871aba46bebd1fdb95c7054afcdb" +checksum = "cd7a4d279762a35b9df97209f6808b95d4fe78547fe2316b4d200a0283960c5a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -660,9 +662,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58b56f1cbe6fd4d0c2573df72868f20ab1c125ca9c9dbce17927a463433a2e57" +checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -700,9 +702,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.7" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fa43bc04a6b2441968faeab56e68da3812f978a670a5db32accbdcafddd12f" +checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -721,9 +723,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.4" +version = "0.60.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ "aws-smithy-types", "bytes", @@ -732,9 +734,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.8" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a7de001a1b9a25601016d8057ea16e31a45fdca3751304c8edf4ad72e706c08" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -762,16 +764,19 @@ dependencies = [ [[package]] name = "aws-smithy-protocol-test" -version = "0.60.7" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31e8279cb24640c7349f2bda6ca818d5fcc85129386bd73c1d0999430d6ddf2" +checksum = "495c940cd5c7232ac3f0945ff559096deadd2fc73e4418a0e98fe5836788bb39" dependencies = [ "assert-json-diff", "aws-smithy-runtime-api", + "base64-simd", + "cbor-diag", "http 0.2.12", "pretty_assertions", "regex-lite", "roxmltree", + "serde_cbor", "serde_json", "thiserror", ] @@ -788,9 +793,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.5.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ac79e9f3a4d576f3cd4a470a0275b138d9e7b11b1cd514a6858ae0a79dd5bb" +checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -802,10 +807,11 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "http-body 1.0.0", - "hyper 0.14.28", + "http-body 1.0.1", + "httparse", + "hyper 0.14.30", "hyper-rustls 0.24.2", - "indexmap 2.2.6", + "indexmap 2.5.0", "once_cell", "pin-project-lite", "pin-utils", @@ -819,9 +825,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.6.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04ec42c2f5c0e7796a2848dde4d9f3bf8ce12ccbb3d5aa40c52fa0cdd61a1c47" +checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -836,9 +842,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.1.9" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf98d97bba6ddaba180f1b1147e202d8fe04940403a95a3f826c790f931bbd1" +checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" dependencies = [ "base64-simd", "bytes", @@ -847,7 +853,7 @@ dependencies = [ "http 0.2.12", "http 1.1.0", "http-body 0.4.6", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "itoa", "num-integer", @@ -872,24 +878,23 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.8" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.2.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a807d90cd50a969b3d95e4e7ad1491fcae13c6e83948d8728363ecc09d66343a" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "http 0.2.12", "rustc_version", "tracing", ] @@ -903,7 +908,7 @@ dependencies = [ "base64 0.22.1", "bytes", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-serde", "query_map", "serde", @@ -921,9 +926,9 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "itoa", "matchit", @@ -938,7 +943,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -954,7 +959,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -976,13 +981,13 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", "serde", "serde_json", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -990,17 +995,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -1066,7 +1071,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -1074,9 +1079,9 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", - "syn 2.0.61", + "syn 2.0.77", "which", ] @@ -1088,9 +1093,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "blake2" @@ -1142,19 +1147,28 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bstr" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ "memchr", "serde", @@ -1168,9 +1182,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.15.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" [[package]] name = "byteorder" @@ -1180,9 +1194,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" dependencies = [ "serde", ] @@ -1242,15 +1256,34 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbor-diag" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" +dependencies = [ + "bs58", + "chrono", + "data-encoding", + "half 2.4.1", + "nom", + "num-bigint", + "num-rational", + "num-traits", + "separator", + "url", + "uuid", +] + [[package]] name = "cc" -version = "1.0.97" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "jobserver", "libc", - "once_cell", + "shlex", ] [[package]] @@ -1302,7 +1335,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -1329,7 +1362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half", + "half 2.4.1", ] [[package]] @@ -1345,9 +1378,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -1356,9 +1389,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.4" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -1366,48 +1399,48 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" dependencies = [ "cc", ] [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "const-oid" @@ -1444,33 +1477,33 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] [[package]] name = "crc32c" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ "rustc_version", ] [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1515,9 +1548,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -1543,9 +1576,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" @@ -1658,14 +1691,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] name = "darling" -version = "0.20.8" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -1673,29 +1706,35 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.8" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 2.0.61", + "strsim", + "syn 2.0.77", ] [[package]] name = "darling_macro" -version = "0.20.8" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.61", + "syn 2.0.77", ] +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + [[package]] name = "data-url" version = "0.3.1" @@ -1724,15 +1763,15 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.17" +version = "0.99.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "convert_case", "proc-macro2", "quote", "rustc_version", - "syn 1.0.109", + "syn 2.0.77", ] [[package]] @@ -1754,9 +1793,9 @@ dependencies = [ [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "ecdsa" @@ -1778,9 +1817,9 @@ checksum = "4a976025474add79730a8df2913b114afd39bc53ce5633e045100aceb6d06bb6" [[package]] name = "either" -version = "1.11.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "elliptic-curve" @@ -1854,9 +1893,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "ff" @@ -1876,9 +1915,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "figment" -version = "0.10.18" +version = "0.10.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d032832d74006f99547004d49410a4b4218e4c33382d56ca3ff89df74f86b953" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" dependencies = [ "atomic", "parking_lot", @@ -1892,9 +1931,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", "miniz_oxide", @@ -1977,7 +2016,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -2033,9 +2072,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -2066,7 +2105,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.6", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -2075,9 +2114,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -2085,13 +2124,19 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "half" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" + [[package]] name = "half" version = "2.4.1" @@ -2130,6 +2175,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2182,7 +2233,7 @@ dependencies = [ "http 0.2.12", "http 1.1.0", "reqwest", - "rustls-pemfile 2.1.2", + "rustls-pemfile 2.1.3", "serde", "serde_json", "tempfile", @@ -2205,14 +2256,14 @@ dependencies = [ "htsget-search", "htsget-test", "http 1.1.0", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "reqwest", "tempfile", "thiserror", "tokio", "tokio-rustls 0.26.0", - "tower", + "tower 0.5.1", "tower-http", "tracing", ] @@ -2231,8 +2282,8 @@ dependencies = [ "rcgen", "regex", "reqwest", - "rustls 0.23.7", - "rustls-pemfile 2.1.2", + "rustls 0.23.13", + "rustls-pemfile 2.1.3", "rustls-pki-types", "serde", "serde_json", @@ -2397,9 +2448,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -2407,14 +2458,14 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -2426,9 +2477,9 @@ checksum = "08a397c49fec283e3d6211adbe480be95aae5f304cfb923e9970e08956d5168a" [[package]] name = "http-serde" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1133cafcce27ea69d35e56b3a8772e265633e04de73c5f4e1afdffc1d19b5419" +checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ "http 1.1.0", "serde", @@ -2436,9 +2487,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -2454,9 +2505,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -2478,16 +2529,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.5", + "h2 0.4.6", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2505,7 +2556,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.28", + "hyper 0.14.30", "log", "rustls 0.21.12", "rustls-native-certs", @@ -2515,46 +2566,47 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", - "rustls 0.22.4", + "rustls 0.23.13", "rustls-pki-types", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tower-service", + "webpki-roots", ] [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", - "hyper 1.3.1", + "http-body 1.0.1", + "hyper 1.4.1", "pin-project-lite", "socket2", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2608,9 +2660,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2635,26 +2687,26 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -2674,6 +2726,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2682,18 +2743,18 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -2711,9 +2772,9 @@ dependencies = [ "futures", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "lambda_runtime", "mime", "percent-encoding", @@ -2736,10 +2797,10 @@ dependencies = [ "bytes", "futures", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "http-serde", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "lambda_runtime_api_client", "pin-project", @@ -2748,7 +2809,7 @@ dependencies = [ "serde_path_to_error", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tracing", ] @@ -2763,12 +2824,12 @@ dependencies = [ "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", "tracing-subscriber", @@ -2782,9 +2843,9 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -2858,25 +2919,25 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.154" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "local-channel" @@ -2907,15 +2968,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" dependencies = [ "hashbrown 0.14.5", ] @@ -2958,9 +3019,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mime" @@ -2970,9 +3031,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", @@ -2986,23 +3047,24 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi 0.3.9", "libc", "log", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3029,9 +3091,9 @@ dependencies = [ [[package]] name = "noodles" -version = "0.80.0" +version = "0.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ea7a4ffa2e3684ce476156d199e388b465d860d702b739f3e39257ded7d174" +checksum = "4f5855c944a96d41a76b13609b9696b2fb7adbc248048f10a4df836edabddd65" dependencies = [ "noodles-bam", "noodles-bcf", @@ -3057,7 +3119,7 @@ dependencies = [ "byteorder", "bytes", "futures", - "indexmap 2.2.6", + "indexmap 2.5.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3067,13 +3129,13 @@ dependencies = [ [[package]] name = "noodles-bcf" -version = "0.59.1" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3206eec367e12e38ff2efb9f11478be5d79a117870f804abc2dec8f27dd10d18" +checksum = "df27249855797f89f13750ff58a914acd4ba97714b3818dff5cb476cef3907c7" dependencies = [ "byteorder", "futures", - "indexmap 2.2.6", + "indexmap 2.5.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3120,7 +3182,7 @@ dependencies = [ "bzip2", "flate2", "futures", - "indexmap 2.2.6", + "indexmap 2.5.0", "md-5", "noodles-bam", "noodles-core", @@ -3139,7 +3201,7 @@ checksum = "a69e79dbc09bd0cb86d29469ed29066e9a163bce6640527b343bdea458144618" dependencies = [ "bit-vec", "byteorder", - "indexmap 2.2.6", + "indexmap 2.5.0", "noodles-bgzf", "noodles-core", "tokio", @@ -3173,12 +3235,12 @@ dependencies = [ [[package]] name = "noodles-gff" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83d0a43629762ce799147e0d60f80293abe731b65740b7a6a92c1279ef88f0b5" +checksum = "8750c06d43f2066ea511a874ed5736470c883f2e39cd0f60f28dd4530d4591b4" dependencies = [ "futures", - "indexmap 2.2.6", + "indexmap 2.5.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3195,7 +3257,7 @@ dependencies = [ "bitflags", "bstr", "futures", - "indexmap 2.2.6", + "indexmap 2.5.0", "lexical-core", "memchr", "noodles-bgzf", @@ -3211,7 +3273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "263e63f58871224d0cd30ffc4a8531fb4f8f8ec686807febde8e19a69673fe01" dependencies = [ "byteorder", - "indexmap 2.2.6", + "indexmap 2.5.0", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -3220,12 +3282,12 @@ dependencies = [ [[package]] name = "noodles-vcf" -version = "0.63.0" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549043d6958379906b1e306804e3c463f24db880da9ab73bd7678592aafb9c89" +checksum = "37459769faa61c655a2f0cfc1f068c69dbaf92eb648250bd7a7e544b6c6664d0" dependencies = [ "futures", - "indexmap 2.2.6", + "indexmap 2.5.0", "memchr", "noodles-bgzf", "noodles-core", @@ -3255,6 +3317,16 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -3271,22 +3343,23 @@ dependencies = [ ] [[package]] -name = "num-traits" -version = "0.2.19" +name = "num-rational" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", + "num-bigint", + "num-integer", + "num-traits", ] [[package]] -name = "num_cpus" -version = "1.16.0" +name = "num-traits" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "hermit-abi", - "libc", + "autocfg", ] [[package]] @@ -3297,9 +3370,9 @@ checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" [[package]] name = "object" -version = "0.32.2" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] @@ -3312,9 +3385,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "opaque-debug" @@ -3353,9 +3426,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -3371,7 +3444,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -3427,7 +3500,7 @@ checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" dependencies = [ "inlinable_string", "pear_codegen", - "yansi 1.0.1", + "yansi", ] [[package]] @@ -3439,7 +3512,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -3475,7 +3548,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -3508,9 +3581,9 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plotters" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -3521,15 +3594,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -3553,18 +3626,21 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", - "yansi 0.5.1", + "yansi", ] [[package]] @@ -3579,12 +3655,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -3613,9 +3689,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -3628,9 +3704,9 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", "version_check", - "yansi 1.0.1", + "yansi", ] [[package]] @@ -3646,19 +3722,67 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.0.0", + "rustls 0.23.13", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash 2.0.0", + "rustls 0.23.13", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -3728,23 +3852,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -3758,20 +3882,20 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] name = "regex-lite" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" @@ -3781,15 +3905,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.1", "bytes", @@ -3797,10 +3921,10 @@ dependencies = [ "futures-core", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", - "hyper-rustls 0.26.0", + "hyper 1.4.1", + "hyper-rustls 0.27.3", "hyper-util", "ipnet", "js-sys", @@ -3809,15 +3933,16 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.22.4", - "rustls-pemfile 2.1.2", + "quinn", + "rustls 0.23.13", + "rustls-pemfile 2.1.3", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tokio-util", "tower-service", "url", @@ -3826,7 +3951,7 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots", - "winreg", + "windows-registry", ] [[package]] @@ -3897,20 +4022,26 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -3933,29 +4064,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki 0.102.4", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls" -version = "0.23.7" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebbbdb961df0ad3f2652da8f3fdc4b36122f568f968f45ad3316f26c025c677b" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -3983,9 +4101,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ "base64 0.22.1", "rustls-pki-types", @@ -3993,9 +4111,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -4009,9 +4127,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "aws-lc-rs", "ring", @@ -4021,9 +4139,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "092474d1a01ea8278f69e6a358998405fae5b8b963ddaeb2b0b04a128bf1dfb0" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" @@ -4033,9 +4151,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "s3s" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e6cdc8002708b435946eec39afa13c43e4288d1de6316a12816e4cfaaa6c2c" +checksum = "fa54e3b4b4791c8c62291516997866b4f265c3fcbfdbcdd0b8da62896fba8bfa" dependencies = [ "arrayvec", "async-trait", @@ -4044,19 +4162,22 @@ dependencies = [ "bytes", "bytestring", "chrono", + "crc32c", "crc32fast", + "digest", "futures", "hex-simd", "hmac", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "httparse", - "hyper 1.3.1", + "hyper 1.4.1", "itoa", "memchr", "mime", "nom", "nugine-rust-utils", + "numeric_cast", "pin-project-lite", "quick-xml", "serde", @@ -4064,8 +4185,10 @@ dependencies = [ "sha1", "sha2", "smallvec", + "sync_wrapper 1.0.1", "thiserror", "time", + "tokio", "tracing", "transform-stream", "urlencoding", @@ -4074,16 +4197,16 @@ dependencies = [ [[package]] name = "s3s-aws" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6de0d6542f9aa260bd219cf69caf4a5d16786114a4fd834df78d6ae85cfaf10f" +checksum = "13db1822175997bba0d1e398a772085b75791aa14e793819bec728ed8b394ff2" dependencies = [ "async-trait", "aws-sdk-s3", "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-types-convert", - "hyper 1.3.1", + "hyper 1.4.1", "s3s", "sync_wrapper 1.0.1", "tracing", @@ -4092,17 +4215,15 @@ dependencies = [ [[package]] name = "s3s-fs" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12416a376cc964d0730718c8e03e2c0e414c1da48c20b1057ca4cad29059b308" +checksum = "b045f5ab67e8536d147ae48e038f6f533717fe5ddb0a68d0ba8f922ce7fb293f" dependencies = [ "async-trait", "base64-simd", "bytes", "chrono", "crc32c", - "crc32fast", - "digest", "futures", "hex-simd", "md-5", @@ -4112,8 +4233,6 @@ dependencies = [ "path-absolutize", "s3s", "serde_json", - "sha1", - "sha2", "thiserror", "time", "tokio", @@ -4144,11 +4263,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4195,9 +4314,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags", "core-foundation", @@ -4208,9 +4327,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -4222,34 +4341,51 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +[[package]] +name = "separator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" + [[package]] name = "serde" -version = "1.0.201" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half 1.8.3", + "serde", +] + [[package]] name = "serde_derive" -version = "1.0.201" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.5.0", "itoa", + "memchr", "ryu", "serde", ] @@ -4276,9 +4412,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" dependencies = [ "serde", ] @@ -4297,15 +4433,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.5.0", "serde", "serde_derive", "serde_json", @@ -4315,14 +4451,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -4434,12 +4570,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -4448,9 +4578,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -4465,9 +4595,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.61" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -4485,17 +4615,21 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4509,22 +4643,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -4580,9 +4714,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -4595,32 +4729,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -4633,33 +4766,22 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.7", + "rustls 0.23.13", "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -4668,9 +4790,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -4682,9 +4804,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", @@ -4694,20 +4816,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.12" +version = "0.22.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", @@ -4730,6 +4852,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tokio", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.5.2" @@ -4740,7 +4877,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "http-range-header", "httpdate", @@ -4757,15 +4894,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -4781,9 +4918,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa069bd1503dd526ee793bb3fce408895136c95fc86d2edb2acf1c646d7f0684" +checksum = "284586dc201db407be8c9d721abad1b3a6dacbbce5cccecd4fd15a37db95ab0d" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -4800,7 +4937,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] @@ -4912,15 +5049,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -4943,9 +5080,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -4960,15 +5097,15 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", ] @@ -4981,9 +5118,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vsimd" @@ -5018,34 +5155,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -5055,9 +5193,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5065,22 +5203,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" @@ -5097,9 +5235,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -5107,9 +5245,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.1" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -5144,11 +5282,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5163,7 +5301,37 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", ] [[package]] @@ -5181,7 +5349,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -5201,18 +5378,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -5223,9 +5400,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -5235,9 +5412,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -5247,15 +5424,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -5265,9 +5442,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -5277,9 +5454,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -5289,9 +5466,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -5301,29 +5478,19 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.8" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "xmlparser" version = "0.13.6" @@ -5339,12 +5506,6 @@ dependencies = [ "lzma-sys", ] -[[package]] -name = "yansi" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" - [[package]] name = "yansi" version = "1.0.1" @@ -5362,53 +5523,54 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.77", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/htsget-actix/Cargo.toml b/htsget-actix/Cargo.toml index 7d7337ec..b8c511b1 100644 --- a/htsget-actix/Cargo.toml +++ b/htsget-actix/Cargo.toml @@ -13,12 +13,12 @@ repository = "https://github.com/umccr/htsget-rs" [features] s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-axum/s3-storage", "htsget-test/s3-storage"] url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-axum/url-storage", "htsget-test/url-storage"] -c4gh-experimental = [ - "htsget-config/c4gh-experimental", - "htsget-search/c4gh-experimental", - "htsget-http/c4gh-experimental", - "htsget-axum/c4gh-experimental", - "htsget-test/c4gh-experimental" +experimental = [ + "htsget-config/experimental", + "htsget-search/experimental", + "htsget-http/experimental", + "htsget-axum/experimental", + "htsget-test/experimental" ] default = [] diff --git a/htsget-actix/README.md b/htsget-actix/README.md index 6ac5d66a..29575f04 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -31,7 +31,7 @@ This crate is used for running a local instance of htsget-rs. It is based on: ## Usage -This application has the same functionality as [htsget-axum]. To use it, following the [htsget-axum][htsget-axum-usage] instructions, and +This application has the same functionality as [htsget-axum]. To use it, following the [htsget-axum][htsget-axum] instructions, and replace any calls to `htsget-axum` with `htsget-actix`. It is recommended to use [htsget-axum] because it better fits with the rest of [htsget-rs]. For example [htsget-actix] @@ -50,7 +50,7 @@ are exposed in the public API. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. ## Benchmarks Benchmarks for this crate written using [Criterion.rs][criterion-rs], and aim to compare the performance of this crate with the @@ -76,6 +76,7 @@ cargo bench -p htsget-axum -- HEAVY [criterion-rs]: https://github.com/bheisler/criterion.rs [htsget-refserver]: https://github.com/ga4gh/htsget-refserver [data-vcf]: ../data/vcf +[htsget-axum]: ../htsget-axum/README.md#usage ## License diff --git a/htsget-actix/src/handlers/get.rs b/htsget-actix/src/handlers/get.rs index d5e2d645..b9569b56 100644 --- a/htsget-actix/src/handlers/get.rs +++ b/htsget-actix/src/handlers/get.rs @@ -17,7 +17,7 @@ use super::handle_response; /// GET request reads endpoint #[instrument(skip(app_state))] -pub async fn reads( +pub async fn reads( request: Query>, path: Path, http_request: HttpRequest, @@ -32,7 +32,7 @@ pub async fn reads( /// GET request variants endpoint #[instrument(skip(app_state))] -pub async fn variants( +pub async fn variants( request: Query>, path: Path, http_request: HttpRequest, diff --git a/htsget-actix/src/handlers/post.rs b/htsget-actix/src/handlers/post.rs index b5796528..bba721f9 100644 --- a/htsget-actix/src/handlers/post.rs +++ b/htsget-actix/src/handlers/post.rs @@ -18,7 +18,7 @@ use super::handle_response; /// POST request reads endpoint #[instrument(skip(app_state))] -pub async fn reads( +pub async fn reads( request: Query>, body: Json, path: Path, @@ -42,7 +42,7 @@ pub async fn reads( /// POST request variants endpoint #[instrument(skip(app_state))] -pub async fn variants( +pub async fn variants( request: Query>, body: Json, path: Path, diff --git a/htsget-actix/src/handlers/service_info.rs b/htsget-actix/src/handlers/service_info.rs index 8c32b967..8df7c6dc 100644 --- a/htsget-actix/src/handlers/service_info.rs +++ b/htsget-actix/src/handlers/service_info.rs @@ -12,7 +12,7 @@ use crate::AppState; /// Gets the JSON to return for a service-info endpoint #[instrument(skip(app_state))] -pub fn get_service_info_json( +pub fn get_service_info_json( app_state: &AppState, endpoint: Endpoint, ) -> impl Responder { @@ -26,14 +26,14 @@ pub fn get_service_info_json( } /// Gets the JSON to return for the reads service-info endpoint -pub async fn reads_service_info( +pub async fn reads_service_info( app_state: Data>, ) -> impl Responder { get_service_info_json(app_state.get_ref(), Endpoint::Reads) } /// Gets the JSON to return for the variants service-info endpoint -pub async fn variants_service_info( +pub async fn variants_service_info( app_state: Data>, ) -> impl Responder { get_service_info_json(app_state.get_ref(), Endpoint::Variants) diff --git a/htsget-actix/src/lib.rs b/htsget-actix/src/lib.rs index 85c80528..323ef1c4 100644 --- a/htsget-actix/src/lib.rs +++ b/htsget-actix/src/lib.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use actix_cors::Cors; use actix_web::dev::Server; use actix_web::{web, App, HttpServer}; @@ -18,19 +16,19 @@ pub mod handlers; /// Represents the actix app state. pub struct AppState { - pub htsget: Arc, + pub htsget: H, pub config_service_info: ServiceInfo, } /// Configure the query server. -pub fn configure_server( +pub fn configure_server( service_config: &mut web::ServiceConfig, htsget: H, config_service_info: ServiceInfo, ) { service_config .app_data(web::Data::new(AppState { - htsget: Arc::new(htsget), + htsget, config_service_info, })) .service( diff --git a/htsget-axum/Cargo.toml b/htsget-axum/Cargo.toml index a5ad8e44..3d081fdd 100644 --- a/htsget-axum/Cargo.toml +++ b/htsget-axum/Cargo.toml @@ -24,11 +24,11 @@ url-storage = [ "htsget-test/url-storage", "htsget-http/url-storage" ] -c4gh-experimental = [ - "htsget-config/c4gh-experimental", - "htsget-search/c4gh-experimental", - "htsget-test/c4gh-experimental", - "htsget-http/c4gh-experimental" +experimental = [ + "htsget-config/experimental", + "htsget-search/experimental", + "htsget-test/experimental", + "htsget-http/experimental" ] default = [] @@ -40,7 +40,7 @@ tower-http = { version = "0.5", features = ["trace", "cors", "fs"] } http = "1" axum = { version = "0.7", features = ["http2"] } axum-extra = { version = "0.9", features = ["erased-json"] } -tower = { version = "0.4", features = ["make"] } +tower = { version = "0.5", features = ["make", "util"] } # Async tokio-rustls = "0.26" diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 00d1cae8..6330d1c1 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -108,7 +108,7 @@ for more details on how to configure this. Run the server with the following to enable Crypt4GH support using the [example config][example-config]: ```sh -cargo run -p htsget-axum --features c4gh-experimental -- --config htsget-config/examples/config-files/c4gh.toml +cargo run -p htsget-axum --features experimental -- --config htsget-config/examples/config-files/c4gh.toml ``` Crypt4GH encrypted byte ranges can be queried: @@ -171,7 +171,7 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. ## License diff --git a/htsget-axum/src/handlers/post.rs b/htsget-axum/src/handlers/post.rs index a77e47d3..e5ad7f2f 100644 --- a/htsget-axum/src/handlers/post.rs +++ b/htsget-axum/src/handlers/post.rs @@ -14,7 +14,7 @@ use crate::server::AppState; use super::handle_response; /// POST request reads endpoint. -pub async fn reads( +pub async fn reads( request: Query>, path: Path, headers: HeaderMap, @@ -27,7 +27,7 @@ pub async fn reads( } /// POST request variants endpoint. -pub async fn variants( +pub async fn variants( request: Query>, path: Path, headers: HeaderMap, diff --git a/htsget-axum/src/server/mod.rs b/htsget-axum/src/server/mod.rs index aa9ab2d8..3fbe7290 100644 --- a/htsget-axum/src/server/mod.rs +++ b/htsget-axum/src/server/mod.rs @@ -36,13 +36,13 @@ use crate::server::ticket::TicketServer; /// Represents the axum app state. #[derive(Debug, Clone)] pub struct AppState { - pub(crate) htsget: Arc, + pub(crate) htsget: H, pub(crate) service_info: ServiceInfo, } impl AppState { /// Create a new app state. - pub fn new(htsget: Arc, service_info: ServiceInfo) -> Self { + pub fn new(htsget: H, service_info: ServiceInfo) -> Self { Self { htsget, service_info, diff --git a/htsget-axum/src/server/ticket.rs b/htsget-axum/src/server/ticket.rs index 5abb5cbc..4d10427a 100644 --- a/htsget-axum/src/server/ticket.rs +++ b/htsget-axum/src/server/ticket.rs @@ -10,7 +10,6 @@ use htsget_config::config::cors::CorsConfig; use htsget_config::config::{Config, ServiceInfo, TicketServerConfig}; use htsget_search::HtsGet; use std::net::SocketAddr; -use std::sync::Arc; use tokio::task::JoinHandle; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; @@ -79,7 +78,7 @@ where .layer(TraceLayer::new_for_http()) .layer(configure_cors(cors)), ) - .with_state(AppState::new(Arc::new(htsget), service_info)) + .with_state(AppState::new(htsget, service_info)) } /// Get the local address the server has bound to. diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index ce9b7812..49dda224 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -13,13 +13,13 @@ repository = "https://github.com/umccr/htsget-rs" [features] s3-storage = [] url-storage = ["dep:reqwest"] -c4gh-experimental = ["dep:crypt4gh"] +experimental = ["dep:crypt4gh"] default = [] [dependencies] thiserror = "1" async-trait = "0.1" -noodles = { version = "0.80", features = ["core"] } +noodles = { version = "0.82", features = ["core"] } serde = { version = "1", features = ["derive"] } serde_with = "3" serde_regex = "1" diff --git a/htsget-config/README.md b/htsget-config/README.md index 43ac8232..5188eeb9 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -491,7 +491,7 @@ See the MinIO deployment [example][minio-deployment] for more information on how ### Crypt4GH There is experimental support for serving [Crypt4GH][c4gh] encrypted files. This can be enabled by compiling with the -`c4gh-experimental` feature flag. +`experimental` feature flag. This allows htsget-rs to read Crypt4GH files and serve them encrypted, directly to the client. In the process of serving the data, htsget-rs will decrypt the headers of the Crypt4GH files and reencrypt them so that the client can read @@ -535,7 +535,7 @@ regex, and changing it by using a substitution string. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. ## License diff --git a/htsget-config/examples/config-files/c4gh.toml b/htsget-config/examples/config-files/c4gh.toml index 22c983a8..c5553a70 100644 --- a/htsget-config/examples/config-files/c4gh.toml +++ b/htsget-config/examples/config-files/c4gh.toml @@ -1,5 +1,5 @@ # An example of running htsget-rs with Crypt4GH enabled. -# Run with `cargo run -p htsget-axum --features c4gh-experimental -- --config htsget-config/examples/config-files/c4gh.toml` +# Run with `cargo run -p htsget-axum --features experimental -- --config htsget-config/examples/config-files/c4gh.toml` ticket_server_addr = "127.0.0.1:8080" data_server_addr = "127.0.0.1:8081" diff --git a/htsget-config/src/storage/object/mod.rs b/htsget-config/src/storage/object/mod.rs index 4a222ebb..409c71c6 100644 --- a/htsget-config/src/storage/object/mod.rs +++ b/htsget-config/src/storage/object/mod.rs @@ -1,10 +1,10 @@ //! Defines the type of object used by storage. //! -#[cfg(feature = "c4gh-experimental")] +#[cfg(feature = "experimental")] pub mod c4gh; -#[cfg(feature = "c4gh-experimental")] +#[cfg(feature = "experimental")] use crate::storage::object::c4gh::C4GHKeys; use serde::{Deserialize, Serialize}; @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; pub enum ObjectType { #[default] Regular, - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] C4GH { #[serde(flatten, skip_serializing)] keys: C4GHKeys, diff --git a/htsget-http/Cargo.toml b/htsget-http/Cargo.toml index 7e984b36..e6863ff7 100644 --- a/htsget-http/Cargo.toml +++ b/htsget-http/Cargo.toml @@ -13,7 +13,7 @@ repository = "https://github.com/umccr/htsget-rs" [features] s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-test/s3-storage"] url-storage = ["htsget-config/url-storage", "htsget-search/url-storage", "htsget-test/url-storage"] -c4gh-experimental = ["htsget-config/c4gh-experimental", "htsget-search/c4gh-experimental", "htsget-test/c4gh-experimental"] +experimental = ["htsget-config/experimental", "htsget-search/experimental", "htsget-test/experimental"] default = [] [dependencies] diff --git a/htsget-http/README.md b/htsget-http/README.md index c0dee5b4..b7969e3b 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -38,7 +38,7 @@ These functions take query and endpoint information, and process it using [htsge This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. [warp]: https://github.com/seanmonstar/warp [htsget-search]: ../htsget-search diff --git a/htsget-http/src/http_core.rs b/htsget-http/src/http_core.rs index 74395c88..33192c2f 100644 --- a/htsget-http/src/http_core.rs +++ b/htsget-http/src/http_core.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use futures::stream::FuturesOrdered; use futures::StreamExt; use tokio::select; @@ -19,7 +17,7 @@ use crate::{ /// consulted [here](https://samtools.github.io/hts-specs/htsget.html) #[instrument(level = "debug", skip_all, ret)] pub async fn get( - searcher: Arc, + searcher: impl HtsGet + Send + Sync + 'static, request: Request, endpoint: Endpoint, ) -> Result { @@ -39,7 +37,7 @@ pub async fn get( /// The parameters can be consulted [here](https://samtools.github.io/hts-specs/htsget.html) #[instrument(level = "debug", skip_all, ret)] pub async fn post( - searcher: Arc, + searcher: impl HtsGet + Clone + Send + Sync + 'static, body: PostRequest, request: Request, endpoint: Endpoint, diff --git a/htsget-http/src/lib.rs b/htsget-http/src/lib.rs index 96eead2a..0ed66aed 100644 --- a/htsget-http/src/lib.rs +++ b/htsget-http/src/lib.rs @@ -83,7 +83,6 @@ fn merge_responses(responses: Vec) -> Option { mod tests { use std::collections::HashMap; use std::path::PathBuf; - use std::sync::Arc; use http::uri::Authority; @@ -270,8 +269,8 @@ mod tests { .join("data") } - fn get_searcher() -> Arc { - Arc::new(HtsGetFromStorage::new(Storage::new( + fn get_searcher() -> impl HtsGet + Clone { + HtsGetFromStorage::new(Storage::new( LocalStorage::new( get_base_path(), ConfigLocalStorage::new( @@ -283,6 +282,6 @@ mod tests { ), ) .unwrap(), - ))) + )) } } diff --git a/htsget-http/src/service_info.rs b/htsget-http/src/service_info.rs index de8d5c1e..5a529dea 100644 --- a/htsget-http/src/service_info.rs +++ b/htsget-http/src/service_info.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use serde::{Deserialize, Serialize}; use tracing::debug; use tracing::instrument; @@ -97,7 +95,7 @@ pub fn get_service_info_with( #[instrument(level = "debug", skip_all)] pub fn get_service_info_json( endpoint: Endpoint, - searcher: Arc, + searcher: impl HtsGet + Send + Sync + 'static, config: &ConfigServiceInfo, ) -> ServiceInfo { debug!(endpoint = ?endpoint,"getting service-info response for endpoint"); diff --git a/htsget-lambda/Cargo.toml b/htsget-lambda/Cargo.toml index 82daff64..00968d12 100644 --- a/htsget-lambda/Cargo.toml +++ b/htsget-lambda/Cargo.toml @@ -13,12 +13,12 @@ repository = "https://github.com/umccr/htsget-rs" [features] s3-storage = ["htsget-axum/s3-storage", "htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage", "htsget-test/s3-storage"] url-storage = ["htsget-axum/url-storage", "htsget-config/url-storage", "htsget-search/url-storage", "htsget-http/url-storage", "htsget-test/url-storage"] -c4gh-experimental = [ - "htsget-axum/c4gh-experimental", - "htsget-config/c4gh-experimental", - "htsget-search/c4gh-experimental", - "htsget-http/c4gh-experimental", - "htsget-test/c4gh-experimental" +experimental = [ + "htsget-axum/experimental", + "htsget-config/experimental", + "htsget-search/experimental", + "htsget-http/experimental", + "htsget-test/experimental" ] default = [] diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index 949393d2..16806fd1 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -46,7 +46,7 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. ## License diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index 95e2ed6e..1aa6e5ad 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -22,10 +22,10 @@ url-storage = [ "htsget-config/url-storage", "htsget-test/url-storage" ] -c4gh-experimental = [ - "htsget-storage/c4gh-experimental", - "htsget-config/c4gh-experimental", - "htsget-test/c4gh-experimental" +experimental = [ + "htsget-storage/experimental", + "htsget-config/experimental", + "htsget-test/experimental" ] default = [] @@ -37,7 +37,7 @@ futures-util = "0.3" async-trait = "0.1" # Noodles -noodles = { version = "0.80", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } +noodles = { version = "0.82", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } # Error control, tracing, config thiserror = "1" diff --git a/htsget-search/README.md b/htsget-search/README.md index 400807ee..3d179331 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -59,7 +59,7 @@ used to process requests. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. ## Minimising Byte Ranges diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index 4ccccf48..53e46c87 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -1,8 +1,6 @@ //! Module providing the search capability using BAM/BAI files //! -use std::sync::Arc; - use async_trait::async_trait; use noodles::bam; use noodles::bam::bai; @@ -21,13 +19,14 @@ use crate::search::{BgzfSearch, Search, SearchAll, SearchReads}; use crate::Class::Body; use crate::HtsGetError; use crate::{Format, Query, Result}; -use htsget_storage::{BytesPosition, Storage, Streamable}; +use htsget_storage::types::BytesPosition; +use htsget_storage::{Storage, Streamable}; type AsyncReader = bam::AsyncReader>; /// Allows searching through bam files. pub struct BamSearch { - storage: Arc, + storage: Storage, } #[async_trait] @@ -95,8 +94,12 @@ impl Search, Index, AsyncReader, Header> for BamS .await } - fn get_storage(&self) -> Arc { - Arc::clone(&self.storage) + fn get_storage(&self) -> &Storage { + &self.storage + } + + fn mut_storage(&mut self) -> &mut Storage { + &mut self.storage } fn get_format(&self) -> Format { @@ -136,24 +139,24 @@ impl SearchReads, Index, AsyncReader, Header> for impl BamSearch { /// Create the bam search. - pub fn new(storage: Arc) -> Self { + pub fn new(storage: Storage) -> Self { Self { storage } } } #[cfg(test)] pub(crate) mod tests { - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; - use htsget_test::http::concat::ConcatResponse; - use std::future::Future; - use super::*; #[cfg(feature = "s3-storage")] use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Body, Class::Header, Headers, HtsGetError::NotFound, Response, Url}; + use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; use htsget_storage::local::LocalStorage; - #[cfg(feature = "c4gh-experimental")] + use htsget_test::http::concat::ConcatResponse; + use std::future::Future; + use std::sync::Arc; + #[cfg(feature = "experimental")] use { crate::from_storage::tests::with_local_storage_c4gh, htsget_storage::c4gh::storage::C4GHStorage, htsget_test::c4gh::get_decryption_keys, @@ -166,7 +169,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_all_reads() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam); let response = search.search(query).await; println!("{response:#?}"); @@ -186,7 +189,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_unmapped_reads() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("*"); let response = search.search(query).await; @@ -213,7 +216,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_reference_name_without_seq_range_chr11() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("11"); let response = search.search(query).await; @@ -237,7 +240,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_reference_name_without_seq_range_chr20() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("20"); let response = search.search(query).await; @@ -265,7 +268,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_reference_name_with_seq_range() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("11") .with_start(5015000) @@ -301,7 +304,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_reference_name_no_end_position() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("11") .with_start(5015000); @@ -330,7 +333,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_many_response_urls() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("11") .with_start(4999976) @@ -365,7 +368,7 @@ pub(crate) mod tests { async fn search_no_gzi() { with_local_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("11") .with_start(5015000) @@ -398,7 +401,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_header() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam).with_class(Header); let response = search.search(query).await; @@ -423,7 +426,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_header_with_no_mapped_reads() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("22"); let response = search.search(query).await; @@ -448,7 +451,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_header_with_non_existent_reference_name() { with_local_storage(|storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("25"); let response = search.search(query).await; @@ -465,7 +468,7 @@ pub(crate) mod tests { async fn search_non_existent_id_reference_name() { with_local_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam); let response = search.search(query).await; assert!(matches!(response, Err(NotFound(_)))); @@ -482,7 +485,7 @@ pub(crate) mod tests { async fn search_non_existent_id_all_reads() { with_local_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("20"); let response = search.search(query).await; @@ -500,7 +503,7 @@ pub(crate) mod tests { async fn search_non_existent_id_header() { with_local_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam).with_class(Header); let response = search.search(query).await; @@ -518,7 +521,7 @@ pub(crate) mod tests { async fn get_header_end_offset() { with_local_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam).with_class(Header); @@ -540,7 +543,7 @@ pub(crate) mod tests { async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam); let response = search.search(query).await; assert!(response.is_err()); @@ -558,7 +561,7 @@ pub(crate) mod tests { async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("20"); let response = search.search(query).await; @@ -577,7 +580,7 @@ pub(crate) mod tests { async fn search_non_existent_id_header_aws() { with_aws_storage_fn( |storage| async move { - let search = BamSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BamSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam).with_class(Header); let response = search.search(query).await; @@ -591,12 +594,12 @@ pub(crate) mod tests { .await } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_all_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = BamSearch::new(Arc::new(Storage::new(storage))); + let mut search = BamSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam); let response = search.search(query).await.unwrap(); @@ -610,12 +613,12 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_all_range_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = BamSearch::new(Arc::new(Storage::new(storage))); + let mut search = BamSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Bam) .with_reference_name("11") .with_start(5015000) diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index 6ebcde94..d3a4a668 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -1,8 +1,6 @@ //! Module providing the search capability using BCF files //! -use std::sync::Arc; - use async_trait::async_trait; use futures_util::stream::FuturesOrdered; use noodles::bcf; @@ -18,13 +16,14 @@ use tracing::{instrument, trace}; use crate::search::{find_first, BgzfSearch, Search}; use crate::{Format, Query, Result}; -use htsget_storage::{BytesPosition, Storage, Streamable}; +use htsget_storage::types::BytesPosition; +use htsget_storage::{Storage, Streamable}; type AsyncReader = bcf::AsyncReader>; /// Allows searching through bcf files. pub struct BcfSearch { - storage: Arc, + storage: Storage, } #[async_trait] @@ -87,8 +86,12 @@ impl Search, Index, AsyncReader, Header> for BcfS Ok(byte_ranges) } - fn get_storage(&self) -> Arc { - self.storage.clone() + fn get_storage(&self) -> &Storage { + &self.storage + } + + fn mut_storage(&mut self) -> &mut Storage { + &mut self.storage } fn get_format(&self) -> Format { @@ -98,18 +101,18 @@ impl Search, Index, AsyncReader, Header> for BcfS impl BcfSearch { /// Create the bcf search. - pub fn new(storage: Arc) -> Self { + pub fn new(storage: Storage) -> Self { Self { storage } } } #[cfg(test)] mod tests { - use std::future::Future; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; use htsget_config::types::Class::Body; use htsget_test::http::concat::ConcatResponse; + use std::future::Future; + use std::sync::Arc; use super::*; #[cfg(feature = "s3-storage")] @@ -118,7 +121,7 @@ mod tests { use crate::search::SearchAll; use crate::{Class::Header, Headers, HtsGetError::NotFound, Response, Url}; use htsget_storage::local::LocalStorage; - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] use { crate::from_storage::tests::with_local_storage_c4gh, htsget_storage::c4gh::storage::C4GHStorage, htsget_test::c4gh::get_decryption_keys, @@ -132,7 +135,7 @@ mod tests { #[tokio::test] async fn search_all_variants() { with_local_storage(|storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "sample1-bcbio-cancer"; let query = Query::new_with_default_request(filename, Format::Bcf); let response = search.search(query).await; @@ -152,7 +155,7 @@ mod tests { #[tokio::test] async fn search_reference_name_without_seq_range() { with_local_storage(|storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "vcf-spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Bcf).with_reference_name("20"); let response = search.search(query).await; @@ -184,7 +187,7 @@ mod tests { #[tokio::test] async fn search_reference_name_no_end_position() { with_local_storage(|storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "sample1-bcbio-cancer"; let query = Query::new_with_default_request(filename, Format::Bcf) .with_reference_name("chrM") @@ -216,7 +219,7 @@ mod tests { #[tokio::test] async fn search_header() { with_local_storage(|storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "vcf-spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Bcf).with_class(Header); let response = search.search(query).await; @@ -242,7 +245,7 @@ mod tests { async fn search_non_existent_id_reference_name() { with_local_storage_fn( |storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf); let response = search.search(query).await; assert!(matches!(response, Err(NotFound(_)))); @@ -259,7 +262,7 @@ mod tests { async fn search_non_existent_id_all_reads() { with_local_storage_fn( |storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_reference_name("chrM"); let response = search.search(query).await; @@ -277,7 +280,7 @@ mod tests { async fn search_non_existent_id_header() { with_local_storage_fn( |storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_class(Header); let response = search.search(query).await; @@ -294,7 +297,7 @@ mod tests { #[tokio::test] async fn search_header_with_non_existent_reference_name() { with_local_storage(|storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_reference_name("chr1"); let response = search.search(query).await; @@ -311,7 +314,7 @@ mod tests { async fn get_header_end_offset() { with_local_storage_fn( |storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_class(Header); @@ -333,7 +336,7 @@ mod tests { async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( |storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf); let response = search.search(query).await; assert!(response.is_err()); @@ -351,7 +354,7 @@ mod tests { async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( |storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_reference_name("chrM"); let response = search.search(query).await; @@ -370,7 +373,7 @@ mod tests { async fn search_non_existent_id_header_aws() { with_aws_storage_fn( |storage| async move { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("vcf-spec-v4.3", Format::Bcf).with_class(Header); let response = search.search(query).await; @@ -384,12 +387,12 @@ mod tests { .await } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_all_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = BcfSearch::new(Arc::new(Storage::new(storage))); + let mut search = BcfSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("sample1-bcbio-cancer", Format::Bcf); let response = search.search(query).await.unwrap(); @@ -403,12 +406,12 @@ mod tests { .await; } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_range_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = BcfSearch::new(Arc::new(Storage::new(storage))); + let mut search = BcfSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("sample1-bcbio-cancer", Format::Bcf) .with_reference_name("chrM") .with_start(150) @@ -428,7 +431,7 @@ mod tests { async fn test_reference_sequence_with_seq_range( storage: Arc>, ) -> Option<(String, ConcatResponse)> { - let search = BcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = BcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "sample1-bcbio-cancer"; let query = Query::new_with_default_request(filename, Format::Bcf) .with_reference_name("chrM") diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index b128d2f8..babe3638 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -23,7 +23,8 @@ use crate::search::{Search, SearchAll, SearchReads}; use crate::Class::Body; use crate::{ConcurrencyError, ParsedHeader}; use crate::{Format, HtsGetError, Query, Result}; -use htsget_storage::{BytesPosition, DataBlock, Storage, Streamable}; +use htsget_storage::types::{BytesPosition, DataBlock}; +use htsget_storage::{Storage, Streamable}; // § 9 End of file container . static CRAM_EOF: &[u8] = &[ @@ -36,7 +37,7 @@ type AsyncReader = cram::AsyncReader>; /// Allows searching through cram files. pub struct CramSearch { - storage: Arc, + storage: Storage, } #[async_trait] @@ -163,8 +164,12 @@ impl Search, Index, AsyncReader, Header> for CramSearch { .await } - fn get_storage(&self) -> Arc { - self.storage.clone() + fn get_storage(&self) -> &Storage { + &self.storage + } + + fn mut_storage(&mut self) -> &mut Storage { + &mut self.storage } fn get_format(&self) -> Format { @@ -174,7 +179,7 @@ impl Search, Index, AsyncReader, Header> for CramSearch { impl CramSearch { /// Create the cram search. - pub fn new(storage: Arc) -> Self { + pub fn new(storage: Storage) -> Self { Self { storage } } @@ -278,7 +283,7 @@ mod tests { use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Header, Headers, HtsGetError::NotFound, Response, Url}; use htsget_storage::local::LocalStorage; - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] use { crate::from_storage::tests::with_local_storage_c4gh, htsget_storage::c4gh::storage::C4GHStorage, htsget_test::c4gh::get_decryption_keys, @@ -291,7 +296,7 @@ mod tests { #[tokio::test] async fn search_all_reads() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram); let response = search.search(query).await; println!("{response:#?}"); @@ -311,7 +316,7 @@ mod tests { #[tokio::test] async fn search_unmapped_reads() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("*"); let response = search.search(query).await; @@ -338,7 +343,7 @@ mod tests { #[tokio::test] async fn search_reference_name_without_seq_range_chr11() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("11"); let response = search.search(query).await; @@ -362,7 +367,7 @@ mod tests { #[tokio::test] async fn search_reference_name_without_seq_range_chr20() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("20"); let response = search.search(query).await; @@ -390,7 +395,7 @@ mod tests { #[tokio::test] async fn search_reference_name_with_seq_range_no_overlap() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("11") .with_start(5000000) @@ -416,7 +421,7 @@ mod tests { #[tokio::test] async fn search_reference_name_with_seq_range_overlap() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("11") .with_start(5000000) @@ -435,7 +440,7 @@ mod tests { #[tokio::test] async fn search_reference_name_with_no_end_position() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("11") .with_start(5000000); @@ -464,7 +469,7 @@ mod tests { #[tokio::test] async fn search_header() { with_local_storage(|storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram).with_class(Header); let response = search.search(query).await; @@ -490,7 +495,7 @@ mod tests { async fn search_non_existent_id_reference_name() { with_local_storage_fn( |storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram); let response = search.search(query).await; assert!(matches!(response, Err(NotFound(_)))); @@ -507,7 +512,7 @@ mod tests { async fn search_non_existent_id_all_reads() { with_local_storage_fn( |storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("20"); let response = search.search(query).await; @@ -525,7 +530,7 @@ mod tests { async fn search_non_existent_id_header() { with_local_storage_fn( |storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram).with_class(Header); let response = search.search(query).await; @@ -544,7 +549,7 @@ mod tests { async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( |storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram); let response = search.search(query).await; assert!(response.is_err()); @@ -562,7 +567,7 @@ mod tests { async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( |storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("20"); let response = search.search(query).await; @@ -581,7 +586,7 @@ mod tests { async fn search_non_existent_id_header_aws() { with_aws_storage_fn( |storage| async move { - let search = CramSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = CramSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram).with_class(Header); let response = search.search(query).await; @@ -595,12 +600,12 @@ mod tests { .await } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_all_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = CramSearch::new(Arc::new(Storage::new(storage))); + let mut search = CramSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram); let response = search.search(query).await.unwrap(); @@ -614,12 +619,12 @@ mod tests { .await; } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_range_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = CramSearch::new(Arc::new(Storage::new(storage))); + let mut search = CramSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("htsnexus_test_NA12878", Format::Cram) .with_reference_name("11") .with_start(5000000) diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index 04d6cc0d..5f5607cc 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -1,8 +1,6 @@ //! Module providing an implementation of the [HtsGet] trait using a [StorageTrait]. //! -use std::sync::Arc; - use async_trait::async_trait; use tracing::debug; use tracing::instrument; @@ -27,20 +25,21 @@ use crate::{Format, HtsGetError}; use htsget_storage::Storage; /// Implementation of the [HtsGet] trait using a [StorageTrait]. +#[derive(Debug, Clone)] pub struct HtsGetFromStorage { - storage_ref: Arc, + storage: Storage, } #[async_trait] impl HtsGet for Vec { - async fn search(&self, query: Query) -> Result { + async fn search(self, query: Query) -> Result { self.as_slice().search(query).await } } #[async_trait] impl HtsGet for &[Resolver] { - async fn search(&self, mut query: Query) -> Result { + async fn search(self, mut query: Query) -> Result { self .resolve_request::(&mut query) .await @@ -51,13 +50,13 @@ impl HtsGet for &[Resolver] { #[async_trait] impl HtsGet for HtsGetFromStorage { #[instrument(level = "debug", skip(self))] - async fn search(&self, query: Query) -> Result { + async fn search(self, query: Query) -> Result { debug!(format = ?query.format(), ?query, "searching {:?}, with query {:?}", query.format(), query); match query.format() { - Format::Bam => BamSearch::new(self.storage()).search(query).await, - Format::Cram => CramSearch::new(self.storage()).search(query).await, - Format::Vcf => VcfSearch::new(self.storage()).search(query).await, - Format::Bcf => BcfSearch::new(self.storage()).search(query).await, + Format::Bam => BamSearch::new(self.into_inner()).search(query).await, + Format::Cram => CramSearch::new(self.into_inner()).search(query).await, + Format::Vcf => VcfSearch::new(self.into_inner()).search(query).await, + Format::Bcf => BcfSearch::new(self.into_inner()).search(query).await, } } } @@ -90,13 +89,15 @@ impl ResolveResponse for HtsGetFromStorage { impl HtsGetFromStorage { pub fn new(storage: Storage) -> Self { - Self { - storage_ref: Arc::new(storage), - } + Self { storage } + } + + pub fn storage(&self) -> &Storage { + &self.storage } - pub fn storage(&self) -> Arc { - Arc::clone(&self.storage_ref) + pub fn into_inner(self) -> Storage { + self.storage } } @@ -105,6 +106,7 @@ pub(crate) mod tests { use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; + use std::sync::Arc; #[cfg(feature = "s3-storage")] use { htsget_storage::s3::S3Storage, htsget_test::aws_mocks::with_s3_test_server, std::fs::create_dir, @@ -117,7 +119,7 @@ pub(crate) mod tests { use htsget_config::types::Class::Body; use htsget_config::types::Scheme::Http; use htsget_storage::local::LocalStorage; - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] use htsget_test::c4gh::decrypt_data; use htsget_test::http::concat::ConcatResponse; @@ -319,7 +321,7 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] pub(crate) async fn with_local_storage_c4gh(test: F) where F: FnOnce(Arc>) -> Fut, diff --git a/htsget-search/src/lib.rs b/htsget-search/src/lib.rs index 27522a5a..4cd848cf 100644 --- a/htsget-search/src/lib.rs +++ b/htsget-search/src/lib.rs @@ -31,7 +31,7 @@ pub mod vcf_search; /// Trait representing a search for either `reads` or `variants` in the HtsGet specification. #[async_trait] pub trait HtsGet { - async fn search(&self, query: Query) -> Result; + async fn search(self, query: Query) -> Result; fn get_supported_formats(&self) -> Vec { vec![Format::Bam, Format::Cram, Format::Vcf, Format::Bcf] diff --git a/htsget-search/src/search.rs b/htsget-search/src/search.rs index 664775cf..126bf1fe 100644 --- a/htsget-search/src/search.rs +++ b/htsget-search/src/search.rs @@ -6,7 +6,6 @@ //! use std::collections::BTreeSet; -use std::sync::Arc; use async_trait::async_trait; use futures::StreamExt; @@ -27,11 +26,10 @@ use htsget_config::types::Class::Header; use crate::ConcurrencyError; use crate::{Class, Class::Body, Format, HtsGetError, Query, Response, Result}; -use htsget_storage::{ - BytesPosition, BytesPositionOptions, HeadOptions, RangeUrlOptions, Storage, StorageTrait, - Streamable, +use htsget_storage::types::{ + BytesPosition, BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions, }; -use htsget_storage::{DataBlock, GetOptions}; +use htsget_storage::{Storage, StorageMiddleware, StorageTrait, Streamable}; // § 4.1.2 End-of-file marker . pub(crate) static BGZF_EOF: &[u8] = &[ @@ -215,7 +213,10 @@ where ) -> Result>; /// Get the storage of this format. - fn get_storage(&self) -> Arc; + fn get_storage(&self) -> &Storage; + + /// Get the mutable storage of this format. + fn mut_storage(&mut self) -> &mut Storage; /// Get the format of this format. fn get_format(&self) -> Format; @@ -223,13 +224,7 @@ where /// Get the position at the end of file marker. #[instrument(level = "trace", skip(self), ret)] async fn position_at_eof(&self, query: &Query) -> Result { - let file_size = self - .get_storage() - .head( - &query.format().fmt_file(query.id()), - HeadOptions::new(query.request().headers()), - ) - .await?; + let file_size = self.file_size(query).await?; Ok( file_size - u64::try_from(self.get_eof_marker().len()) @@ -254,7 +249,7 @@ where } /// Search based on the query. - async fn search(&self, query: Query) -> Result { + async fn search(&mut self, query: Query) -> Result { match query.class() { Body => { let format = self.get_format(); @@ -266,12 +261,14 @@ where ))); } + let index = self.read_index(&query).await?; + let header_end = self.get_header_end_offset(&index).await?; + + self.preprocess(&query, header_end).await?; + let mut byte_ranges = match query.reference_name().as_ref() { None => self.get_byte_ranges_for_all(&query).await?, Some(reference_name) => { - let index = self.read_index(&query).await?; - - let header_end = self.get_header_end_offset(&index).await?; let (header, mut reader) = self.get_header(&query, header_end).await?; let mut byte_ranges = self @@ -293,20 +290,14 @@ where } }; - let file_size = self - .get_storage() - .head( - &query.format().fmt_file(query.id()), - HeadOptions::new(query.request().headers()), - ) - .await?; + let file_size = self.file_size(&query).await?; if let Some(eof) = self.get_eof_byte_positions(file_size) { byte_ranges.push(eof?); } let blocks = self .get_storage() - .update_byte_positions( + .postprocess( &query.format().fmt_file(query.id()), BytesPositionOptions::new(byte_ranges, query.request().headers()), ) @@ -315,18 +306,11 @@ where self.build_response(&query, blocks).await } Class::Header => { - // Check to see if the key exists. - self - .get_storage() - .head( - &query.format().fmt_file(query.id()), - HeadOptions::new(query.request().headers()), - ) - .await?; - let index = self.read_index(&query).await?; - let header_end = self.get_header_end_offset(&index).await?; + + self.preprocess(&query, header_end).await?; + let (_, mut reader) = self.get_header(&query, header_end).await?; let header_byte_ranges = self @@ -335,7 +319,7 @@ where let blocks = self .get_storage() - .update_byte_positions( + .postprocess( &query.format().fmt_file(query.id()), BytesPositionOptions::new(vec![header_byte_ranges], query.request().headers()), ) @@ -346,42 +330,62 @@ where } } + async fn preprocess(&mut self, query: &Query, header_end: u64) -> Result<()> { + Ok( + self + .mut_storage() + .preprocess( + &query.format().fmt_file(query.id()), + GetOptions::new( + BytesPosition::default().with_end(header_end), + query.request().headers(), + ), + ) + .await?, + ) + } + + async fn file_size(&self, query: &Query) -> Result { + Ok( + self + .get_storage() + .head( + &query.format().fmt_file(query.id()), + HeadOptions::new(query.request().headers()), + ) + .await?, + ) + } + /// Build the response from the query using urls. #[instrument(level = "trace", skip(self, byte_ranges))] async fn build_response(&self, query: &Query, byte_ranges: Vec) -> Result { trace!("building response"); - let mut storage_futures = FuturesOrdered::new(); + let mut urls = vec![]; + let storage = self.get_storage(); + for block in DataBlock::update_classes(byte_ranges) { match block { DataBlock::Range(range) => { trace!(range = ?range, "range"); - let storage = self.get_storage(); let query_owned = query.clone(); - storage_futures.push_back(tokio::spawn(async move { + urls.push( storage .range_url( &query_owned.format().fmt_file(query_owned.id()), RangeUrlOptions::new(range, query_owned.request().headers()), ) - .await - })); + .await?, + ); } DataBlock::Data(data, class) => { let data_url = self.get_storage().data_url(data, class); - storage_futures.push_back(tokio::spawn(async move { Ok(data_url) })); + urls.push(data_url); } } } - let mut urls = Vec::new(); - loop { - select! { - Some(next) = storage_futures.next() => urls.push(next.map_err(ConcurrencyError::new).map_err(HtsGetError::from)?.map_err(HtsGetError::from)?), - else => break - } - } - Ok(Response::new(query.format(), urls)) } diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index d07a5695..d5c3acf6 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -1,8 +1,6 @@ //! Module providing the search capability using VCF files //! -use std::sync::Arc; - use async_trait::async_trait; use futures_util::stream::FuturesOrdered; use noodles::bgzf; @@ -22,13 +20,14 @@ use htsget_config::types::HtsGetError; use crate::search::{find_first, BgzfSearch, Search}; use crate::{Format, Query, Result}; -use htsget_storage::{BytesPosition, Storage, Streamable}; +use htsget_storage::types::BytesPosition; +use htsget_storage::{Storage, Streamable}; type AsyncReader = vcf::AsyncReader>; /// Allows searching through vcf files. pub struct VcfSearch { - storage: Arc, + storage: Storage, } #[async_trait] @@ -98,8 +97,12 @@ impl Search, Index, AsyncReader, Header> for VcfS Ok(byte_ranges) } - fn get_storage(&self) -> Arc { - self.storage.clone() + fn get_storage(&self) -> &Storage { + &self.storage + } + + fn mut_storage(&mut self) -> &mut Storage { + &mut self.storage } fn get_format(&self) -> Format { @@ -109,18 +112,18 @@ impl Search, Index, AsyncReader, Header> for VcfS impl VcfSearch { /// Create the vcf search. - pub fn new(storage: Arc) -> Self { + pub fn new(storage: Storage) -> Self { Self { storage } } } #[cfg(test)] pub(crate) mod tests { - use std::future::Future; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; use htsget_config::types::Class::Body; use htsget_test::http::concat::ConcatResponse; + use std::future::Future; + use std::sync::Arc; use super::*; #[cfg(feature = "s3-storage")] @@ -129,7 +132,7 @@ pub(crate) mod tests { use crate::search::SearchAll; use crate::{Class::Header, Headers, HtsGetError::NotFound, Response, Url}; use htsget_storage::local::LocalStorage; - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] use { crate::from_storage::tests::with_local_storage_c4gh, htsget_storage::c4gh::storage::C4GHStorage, htsget_test::c4gh::get_decryption_keys, @@ -143,7 +146,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_all_variants() { with_local_storage(|storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "sample1-bcbio-cancer"; let query = Query::new_with_default_request(filename, Format::Vcf); let response = search.search(query).await; @@ -163,7 +166,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_reference_name_without_seq_range() { with_local_storage(|storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Vcf).with_reference_name("20"); let response = search.search(query).await; @@ -193,7 +196,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_reference_name_no_end_position() { with_local_storage(|storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "sample1-bcbio-cancer"; let query = Query::new_with_default_request(filename, Format::Vcf) .with_reference_name("chrM") @@ -229,7 +232,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_header() { with_local_storage(|storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "spec-v4.3"; let query = Query::new_with_default_request(filename, Format::Vcf).with_class(Header); let response = search.search(query).await; @@ -255,7 +258,7 @@ pub(crate) mod tests { async fn search_non_existent_id_reference_name() { with_local_storage_fn( |storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf); let response = search.search(query).await; assert!(matches!(response, Err(NotFound(_)))); @@ -272,7 +275,7 @@ pub(crate) mod tests { async fn search_non_existent_id_all_reads() { with_local_storage_fn( |storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_reference_name("chrM"); let response = search.search(query).await; @@ -290,7 +293,7 @@ pub(crate) mod tests { async fn search_non_existent_id_header() { with_local_storage_fn( |storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_class(Header); let response = search.search(query).await; assert!(matches!(response, Err(NotFound(_)))); @@ -306,7 +309,7 @@ pub(crate) mod tests { #[tokio::test] async fn search_header_with_non_existent_reference_name() { with_local_storage(|storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_reference_name("chr1"); let response = search.search(query).await; @@ -323,7 +326,7 @@ pub(crate) mod tests { async fn get_header_end_offset() { with_local_storage_fn( |storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_class(Header); let index = search.read_index(&query).await.unwrap(); @@ -344,7 +347,7 @@ pub(crate) mod tests { async fn search_non_existent_id_reference_name_aws() { with_aws_storage_fn( |storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf); let response = search.search(query).await; assert!(response.is_err()); @@ -362,7 +365,7 @@ pub(crate) mod tests { async fn search_non_existent_id_all_reads_aws() { with_aws_storage_fn( |storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_reference_name("chrM"); let response = search.search(query).await; @@ -381,7 +384,7 @@ pub(crate) mod tests { async fn search_non_existent_id_header_aws() { with_aws_storage_fn( |storage| async move { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf).with_class(Header); let response = search.search(query).await; assert!(response.is_err()); @@ -394,12 +397,12 @@ pub(crate) mod tests { .await } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_all_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = VcfSearch::new(Arc::new(Storage::new(storage))); + let mut search = VcfSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf); let response = search.search(query).await.unwrap(); @@ -410,12 +413,12 @@ pub(crate) mod tests { .await; } - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] #[tokio::test] async fn search_all_range_c4gh() { with_local_storage_c4gh(|storage| async move { let storage = C4GHStorage::new(get_decryption_keys(), Arc::try_unwrap(storage).unwrap()); - let search = VcfSearch::new(Arc::new(Storage::new(storage))); + let mut search = VcfSearch::new(Storage::new(storage)); let query = Query::new_with_default_request("spec-v4.3", Format::Vcf) .with_reference_name("20") .with_start(150) @@ -432,7 +435,7 @@ pub(crate) mod tests { async fn test_reference_name_with_seq_range( storage: Arc>, ) -> Option<(String, ConcatResponse)> { - let search = VcfSearch::new(Arc::new(Storage::new(Arc::try_unwrap(storage).unwrap()))); + let mut search = VcfSearch::new(Storage::new(Arc::try_unwrap(storage).unwrap())); let filename = "sample1-bcbio-cancer"; let query = Query::new_with_default_request(filename, Format::Vcf) .with_reference_name("chrM") diff --git a/htsget-storage/Cargo.toml b/htsget-storage/Cargo.toml index 92ac0459..dfe3b28b 100644 --- a/htsget-storage/Cargo.toml +++ b/htsget-storage/Cargo.toml @@ -25,7 +25,7 @@ url-storage = [ "htsget-config/url-storage", "htsget-test/url-storage" ] -c4gh-experimental = ["dep:crypt4gh", "dep:bincode", "htsget-config/c4gh-experimental", "htsget-test/c4gh-experimental"] +experimental = ["dep:crypt4gh", "dep:bincode", "htsget-config/experimental", "htsget-test/experimental"] default = [] [dependencies] diff --git a/htsget-storage/README.md b/htsget-storage/README.md index 7eb64f2d..23bf7bed 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -49,7 +49,7 @@ and [url] modules implement the `Storage` functionality. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. [local]: src/local.rs [s3]: src/s3.rs diff --git a/htsget-storage/src/c4gh/edit.rs b/htsget-storage/src/c4gh/edit.rs index f4660dad..4107cbb8 100644 --- a/htsget-storage/src/c4gh/edit.rs +++ b/htsget-storage/src/c4gh/edit.rs @@ -5,7 +5,7 @@ use crate::c4gh::DeserializedHeader; use crate::error::{Result, StorageError}; use crypt4gh::error::Crypt4GHError; use crypt4gh::error::Crypt4GHError::InvalidPacketType; -use crypt4gh::header::{encrypt, make_packet_data_edit_list, make_packet_data_enc}; +use crypt4gh::header::{encrypt, make_packet_data_edit_list, make_packet_data_enc, HeaderInfo}; use crypt4gh::Keys; use std::collections::HashSet; use tokio::io; @@ -77,7 +77,7 @@ pub struct EditHeader<'a> { unencrypted_positions: Vec, clamped_positions: Vec, keys: &'a [Keys], - current_header: &'a mut DeserializedHeader, + current_header: &'a DeserializedHeader, } impl<'a> EditHeader<'a> { @@ -86,7 +86,7 @@ impl<'a> EditHeader<'a> { unencrypted_positions: Vec, clamped_positions: Vec, keys: &'a [Keys], - current_header: &'a mut DeserializedHeader, + current_header: &'a DeserializedHeader, ) -> Self { Self { unencrypted_positions, @@ -202,9 +202,18 @@ impl<'a> EditHeader<'a> { ) } - self.current_header.header_info.packets_count += 1 + header_packets.len() as u32; - let header_info_bytes = - bincode::serialize(&self.current_header.header_info).map_err(|_| InvalidPacketType)?; + let header_info = &self.current_header.header_info; + + let mut current_len = header_info.packets_count; + current_len += 1 + header_packets.len() as u32; + + let header_info = HeaderInfo { + magic_number: header_info.magic_number, + version: header_info.version, + packets_count: current_len, + }; + + let header_info_bytes = bincode::serialize(&header_info).map_err(|_| InvalidPacketType)?; Ok( ( @@ -239,7 +248,7 @@ mod tests { test_unencrypted_positions(), test_clamped_positions(), &keys, - &mut DeserializedHeader::from_buffer(&mut buf, &keys).unwrap(), + &DeserializedHeader::from_buffer(&mut buf, &keys).unwrap(), ) .create_edit_list(); diff --git a/htsget-storage/src/c4gh/mod.rs b/htsget-storage/src/c4gh/mod.rs index 0d2e476f..44a1c893 100644 --- a/htsget-storage/src/c4gh/mod.rs +++ b/htsget-storage/src/c4gh/mod.rs @@ -4,9 +4,10 @@ use crypt4gh::error::Crypt4GHError; use crypt4gh::header::{DecryptedHeaderPackets, HeaderInfo}; -use crypt4gh::{header, Keys}; +use crypt4gh::{body_decrypt, body_decrypt_parts, header, Keys, WriteInfo}; use std::cmp::min; -use std::io::Read; +use std::io; +use std::io::{BufWriter, Cursor, Read}; mod edit; pub mod storage; @@ -24,6 +25,23 @@ pub struct DeserializedHeader { pub(crate) session_keys: Vec>, pub(crate) header_size: u64, pub(crate) contains_edit_list: bool, + pub(crate) decrypted_stream: Vec, +} + +impl Clone for DeserializedHeader { + fn clone(&self) -> Self { + Self { + header_info: HeaderInfo { + magic_number: self.header_info.magic_number, + version: self.header_info.version, + packets_count: self.header_info.packets_count, + }, + session_keys: self.session_keys.clone(), + header_size: self.header_size, + contains_edit_list: self.contains_edit_list, + decrypted_stream: self.decrypted_stream.clone(), + } + } } impl DeserializedHeader { @@ -33,12 +51,14 @@ impl DeserializedHeader { session_keys: Vec>, header_size: u64, contains_edit_list: bool, + decrypted_stream: Vec, ) -> Self { Self { header_info, session_keys, header_size, contains_edit_list, + decrypted_stream, } } @@ -53,7 +73,7 @@ impl DeserializedHeader { read_buffer .read_exact(&mut temp_buf) .map_err(|e| Crypt4GHError::ReadHeaderError(e.into()))?; - let header_info: header::HeaderInfo = header::deconstruct_header_info(&temp_buf)?; + let header_info: HeaderInfo = header::deconstruct_header_info(&temp_buf)?; let mut bytes = vec![]; let mut header_lengths = 0; @@ -95,13 +115,35 @@ impl DeserializedHeader { let header_size = 16 + header_lengths; let contains_header = edit_list_packet.is_some(); + let mut writer = BufWriter::new(Cursor::new(vec![])); + let mut write_info = WriteInfo::new(0, None, &mut writer); + + match edit_list_packet { + None => body_decrypt(read_buffer, &session_keys, &mut write_info, 0)?, + Some(edit_list_content) => body_decrypt_parts( + read_buffer, + session_keys.clone(), + write_info, + edit_list_content, + )?, + } + + let data = writer + .into_inner() + .map_err(|err| Crypt4GHError::IoError(io::Error::other(err)))? + .into_inner(); + Ok(DeserializedHeader::new( header_info, session_keys, header_size as u64, contains_header, + data, )) } + + /// Decrypt the + pub fn decrypt_stream() {} } /// Convert an encrypted file position to an unencrypted position if the header length is known. diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index 34df0116..8ec057e7 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -8,19 +8,19 @@ use crate::c4gh::{ }; use crate::error::StorageError::{InternalError, IoError}; use crate::error::{Result, StorageError}; +use crate::types::BytesPosition; use crate::{ - BytesPosition, BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions, + BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions, StorageMiddleware, StorageTrait, Streamable, }; use async_trait::async_trait; use crypt4gh::error::Crypt4GHError; -use crypt4gh::{decrypt, Keys}; +use crypt4gh::Keys; use htsget_config::types::{Class, Format, Url}; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; use std::io; -use std::io::{BufReader, BufWriter, Cursor}; -use std::sync::{Arc, Mutex, PoisonError}; +use std::io::{BufReader, Cursor}; use tokio::io::AsyncReadExt; /// Max C4GH header size in bytes. Supports 50 regular sized encrypted packets. 16 + (108 * 50). @@ -28,7 +28,7 @@ const MAX_C4GH_HEADER_SIZE: u64 = 5416; /// This represents the state that the C4GHStorage needs to save, like the file sizes and header /// sizes. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct C4GHState { encrypted_file_size: u64, unencrypted_file_size: u64, @@ -40,10 +40,17 @@ pub struct C4GHState { pub struct C4GHStorage { keys: Vec, inner: Box, - // Need to have a Mutex so that we can alter the state from a &self reference. - // This is a bit lazy, the proper solution would be to pass around mutable state as a parameter - // or make `StorageTrait` mutable, and synchronise somewhere else. - state: Arc>>, + state: HashMap, +} + +impl Clone for C4GHStorage { + fn clone(&self) -> Self { + Self { + keys: self.keys.clone(), + inner: self.inner.clone_box(), + state: self.state.clone(), + } + } } impl Debug for C4GHStorage { @@ -73,46 +80,45 @@ impl C4GHStorage { return self.inner.get(key, options).await; } - let mut buf = vec![]; - self - .inner - .get(&Self::format_key(key), options) - .await? - .read_to_end(&mut buf) - .await?; - - let mut reader = BufReader::new(Cursor::new(buf)); - let mut writer = BufWriter::new(Cursor::new(vec![])); - - decrypt(&self.keys, &mut reader, &mut writer, 0, None, &None) - .map_err(|err| IoError("Crypt4GH".to_string(), io::Error::other(err)))?; + let data = self + .state + .get(&Self::format_key(key)) + .ok_or_else(|| InternalError("missing key from state".to_string()))? + .clone(); - let data = writer - .into_inner() - .map_err(|err| IoError("Writer".to_string(), io::Error::other(err)))? - .into_inner(); - Ok(Streamable::from_async_read(Cursor::new(data))) + Ok(Streamable::from_async_read(Cursor::new( + data.deserialized_header.decrypted_stream, + ))) } /// Get the size of the unencrypted object and update state. - pub async fn head_object_with_state(&self, key: &str, options: HeadOptions<'_>) -> Result { + pub async fn preprocess_for_state( + &mut self, + key: &str, + mut options: GetOptions<'_>, + ) -> Result { + if Format::is_index(key) { + return self.inner.head(key, (&options).into()).await; + } + + let key = Self::format_key(key); + // Get the file size. - let encrypted_file_size = self - .inner - .head(&Self::format_key(key), options.clone()) - .await?; + let encrypted_file_size = self.inner.head(&key, (&options).into()).await?; + + let end = options + .range + .end + .unwrap_or_default() + .checked_add(MAX_C4GH_HEADER_SIZE) + .ok_or_else(|| InternalError("overflow getting header".to_string()))?; + options.range = options.range.with_end(end); // Also need to determine the header size. let mut buf = vec![]; self .inner - .get( - &Self::format_key(key), - GetOptions::new( - BytesPosition::default().with_end(MAX_C4GH_HEADER_SIZE), - options.request_headers(), - ), - ) + .get(&key, options) .await? .read_to_end(&mut buf) .await?; @@ -128,8 +134,8 @@ impl C4GHStorage { unencrypted_file_size, deserialized_header, }; - let mut header_sizes = self.state.lock()?; - header_sizes.insert(key.to_string(), state); + + self.state.insert(key, state); Ok(unencrypted_file_size) } @@ -140,9 +146,9 @@ impl C4GHStorage { key: &str, options: BytesPositionOptions<'_>, ) -> Result> { - let mut state = self.state.lock()?; - let state = state - .get_mut(key) + let state = self + .state + .get(&Self::format_key(key)) .ok_or_else(|| InternalError("missing key from state".to_string()))?; let default_start = |pos: &BytesPosition| pos.start.unwrap_or_default(); @@ -199,7 +205,7 @@ impl C4GHStorage { unencrypted_positions, clamped_positions, &self.keys, - &mut state.deserialized_header, + &state.deserialized_header, ) .reencrypt_header()? .into_inner(); @@ -227,6 +233,22 @@ impl C4GHStorage { } } +#[async_trait] +impl StorageMiddleware for C4GHStorage { + async fn preprocess(&mut self, key: &str, options: GetOptions<'_>) -> Result<()> { + self.preprocess_for_state(key, options).await?; + Ok(()) + } + + async fn postprocess( + &self, + key: &str, + positions_options: BytesPositionOptions<'_>, + ) -> Result> { + self.compute_data_blocks(key, positions_options).await + } +} + #[async_trait] impl StorageTrait for C4GHStorage { /// Get the Crypt4GH file at the location of the key. @@ -240,17 +262,14 @@ impl StorageTrait for C4GHStorage { } /// Get the size of the underlying file and the encrypted file, updating any state. - async fn head(&self, key: &str, options: HeadOptions<'_>) -> Result { - self.head_object_with_state(key, options).await - } - - /// Update encrypted positions. - async fn update_byte_positions( - &self, - key: &str, - positions_options: BytesPositionOptions<'_>, - ) -> Result> { - self.compute_data_blocks(key, positions_options).await + async fn head(&self, key: &str, _options: HeadOptions<'_>) -> Result { + Ok( + self + .state + .get(&Self::format_key(key)) + .ok_or_else(|| InternalError("failed to call preprocess".to_string()))? + .unencrypted_file_size, + ) } } @@ -260,8 +279,171 @@ impl From for StorageError { } } -impl From> for StorageError { - fn from(err: PoisonError) -> Self { - InternalError(err.to_string()) +#[cfg(test)] +mod tests { + use super::*; + use crate::local::tests::with_local_storage; + use htsget_config::types::Headers; + use htsget_test::c4gh::{encrypt_data, get_decryption_keys}; + use std::future::Future; + use tokio::fs::File; + use tokio::io::AsyncWriteExt; + + #[tokio::test] + async fn test_preprocess() { + with_c4gh_storage(|mut storage| async move { + storage + .preprocess( + "key", + GetOptions::new_with_default_range(&Default::default()), + ) + .await + .unwrap(); + + let state = storage.state.get("key.c4gh").unwrap(); + + assert_eq!(state.unencrypted_file_size, 6); + assert_eq!(state.encrypted_file_size, 158); + assert_eq!(state.deserialized_header.header_info.packets_count, 1); + }) + .await; + } + + #[tokio::test] + async fn test_get() { + with_c4gh_storage(|mut storage| async move { + let headers = Default::default(); + let options = GetOptions::new_with_default_range(&headers); + storage.preprocess("key", options.clone()).await.unwrap(); + let mut object = vec![]; + + storage + .get("key", options) + .await + .unwrap() + .read_to_end(&mut object) + .await + .unwrap(); + assert_eq!(object, b"value1"); + }) + .await; + } + + #[tokio::test] + async fn test_head() { + with_c4gh_storage(|mut storage| async move { + let headers = Default::default(); + let options = GetOptions::new_with_default_range(&headers); + storage.preprocess("key", options.clone()).await.unwrap(); + + let size = storage + .head("key", HeadOptions::new(&headers)) + .await + .unwrap(); + assert_eq!(size, 6); + }) + .await; + } + + #[tokio::test] + async fn test_postprocess() { + with_c4gh_storage(|mut storage| async move { + let headers = Default::default(); + let options = GetOptions::new_with_default_range(&headers); + storage.preprocess("key", options.clone()).await.unwrap(); + + let blocks = storage + .postprocess( + "key", + BytesPositionOptions::new( + vec![BytesPosition::default().with_start(0).with_end(6)], + &headers, + ), + ) + .await + .unwrap(); + + assert_eq!( + blocks[0], + DataBlock::Data( + vec![99, 114, 121, 112, 116, 52, 103, 104, 1, 0, 0, 0, 3, 0, 0, 0], + Some(Class::Header) + ) + ); + assert_eq!( + blocks[1], + DataBlock::Range(BytesPosition::new(Some(16), Some(124), None)) + ); + assert_eq!( + blocks[3], + DataBlock::Range(BytesPosition::new(Some(124), Some(158), None)) + ); + }) + .await; + } + + #[tokio::test] + async fn test_range_url() { + with_c4gh_storage(|mut storage| async move { + let headers = Default::default(); + let options = GetOptions::new_with_default_range(&headers); + storage.preprocess("key", options.clone()).await.unwrap(); + + let blocks = storage + .postprocess( + "key", + BytesPositionOptions::new( + vec![BytesPosition::default().with_start(0).with_end(6)], + &headers, + ), + ) + .await + .unwrap(); + + if let DataBlock::Range(range) = blocks.last().unwrap() { + let url = storage + .range_url("key", RangeUrlOptions::new(range.clone(), &headers)) + .await + .unwrap(); + let expected = Url::new("http://127.0.0.1:8081/data/key.c4gh") + .with_headers(Headers::default().with_header("Range", "bytes=124-157")); + + assert_eq!(url, expected); + } + }) + .await; + } + + pub(crate) async fn with_c4gh_storage(test: F) + where + F: FnOnce(C4GHStorage) -> Fut, + Fut: Future, + { + with_local_storage(|storage| async move { + let mut data = vec![]; + StorageTrait::get( + &storage, + "folder/../key1", + GetOptions::new_with_default_range(&Default::default()), + ) + .await + .unwrap() + .read_to_end(&mut data) + .await + .unwrap(); + + let data = encrypt_data(&data); + + let key = "key.c4gh"; + File::create(storage.base_path().join(key)) + .await + .unwrap() + .write_all(&data) + .await + .unwrap(); + + test(C4GHStorage::new(get_decryption_keys(), storage)).await; + }) + .await; } } diff --git a/htsget-storage/src/error.rs b/htsget-storage/src/error.rs index 900dbc7f..2f02d788 100644 --- a/htsget-storage/src/error.rs +++ b/htsget-storage/src/error.rs @@ -81,3 +81,21 @@ impl From for StorageError { Self::IoError("io error".to_string(), error) } } + +#[cfg(test)] +mod tests { + use super::*; + use htsget_config::types::HtsGetError; + + #[test] + fn htsget_error_from_storage_not_found() { + let result = HtsGetError::from(StorageError::KeyNotFound("error".to_string())); + assert!(matches!(result, HtsGetError::NotFound(_))); + } + + #[test] + fn htsget_error_from_storage_invalid_key() { + let result = HtsGetError::from(StorageError::InvalidKey("error".to_string())); + assert!(matches!(result, HtsGetError::NotFound(_))); + } +} diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index 2414a8e5..0c4d0714 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -17,36 +17,34 @@ use htsget_config::storage::local::LocalStorage as LocalStorageConfig; use htsget_config::storage::s3::S3Storage as S3StorageConfig; #[cfg(feature = "url-storage")] use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; -use http::{uri, HeaderMap}; +use http::uri; use pin_project_lite::pin_project; -use std::cmp::Ordering; use std::fmt; -use std::fmt::{Debug, Display, Formatter}; -use std::num::ParseIntError; +use std::fmt::{Debug, Formatter}; use std::pin::Pin; -use std::str::FromStr; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, ReadBuf}; -use tracing::instrument; -#[cfg(feature = "c4gh-experimental")] +#[cfg(feature = "experimental")] use crate::c4gh::storage::C4GHStorage; use crate::error::Result; use crate::error::StorageError; use crate::local::LocalStorage; #[cfg(feature = "s3-storage")] use crate::s3::S3Storage; +use crate::types::{BytesPositionOptions, DataBlock, GetOptions, HeadOptions, RangeUrlOptions}; #[cfg(feature = "url-storage")] use crate::url::UrlStorage; use htsget_config::storage::object::ObjectType; use htsget_config::types::Scheme; -#[cfg(feature = "c4gh-experimental")] +#[cfg(feature = "experimental")] pub mod c4gh; pub mod error; pub mod local; #[cfg(feature = "s3-storage")] pub mod s3; +pub mod types; #[cfg(feature = "url-storage")] pub mod url; @@ -82,12 +80,35 @@ pub struct Storage { inner: Box, } +impl Clone for Storage { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone_box(), + } + } +} + impl Debug for Storage { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "Storage") } } +#[async_trait] +impl StorageMiddleware for Storage { + async fn preprocess(&mut self, _key: &str, _options: GetOptions<'_>) -> Result<()> { + self.inner.preprocess(_key, _options).await + } + + async fn postprocess( + &self, + key: &str, + positions_options: BytesPositionOptions<'_>, + ) -> Result> { + self.inner.postprocess(key, positions_options).await + } +} + #[async_trait] impl StorageTrait for Storage { async fn get(&self, key: &str, options: GetOptions<'_>) -> Result { @@ -105,17 +126,6 @@ impl StorageTrait for Storage { fn data_url(&self, data: Vec, class: Option) -> Url { self.inner.data_url(data, class) } - - async fn update_byte_positions( - &self, - key: &str, - positions_options: BytesPositionOptions<'_>, - ) -> Result> { - self - .inner - .update_byte_positions(key, positions_options) - .await - } } impl Storage { @@ -124,7 +134,7 @@ impl Storage { let storage = LocalStorage::new(config.local_path(), config.clone())?; match config.object_type() { ObjectType::Regular => Ok(Storage::new(storage)), - #[cfg(feature = "c4gh-experimental")] + #[cfg(feature = "experimental")] ObjectType::C4GH { keys } => Ok(Storage::new(C4GHStorage::new( keys.clone().into_inner(), storage, @@ -170,7 +180,7 @@ impl Storage { /// A Storage represents some kind of object based storage (either locally or in the cloud) /// that can be used to retrieve files for alignments, variants or its respective indexes. #[async_trait] -pub trait StorageTrait { +pub trait StorageTrait: StorageMiddleware + StorageClone { /// Get the object using the key. async fn get(&self, key: &str, options: GetOptions<'_>) -> Result; @@ -189,9 +199,33 @@ pub trait StorageTrait { )) .set_class(class) } +} + +/// Allow the `StorageTrait` to be cloned. This allows cloning a dynamic trait inside a Box. +/// See https://crates.io/crates/dyn-clone for a similar pattern. +pub trait StorageClone { + fn clone_box(&self) -> Box; +} + +impl StorageClone for T +where + T: StorageTrait + Send + Sync + Clone + 'static, +{ + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +/// A middleware trait which related to transforming or processing data returned from `StorageTrait`. +#[async_trait] +pub trait StorageMiddleware { + /// Preprocess any required state before it is requested by `StorageTrait`. + async fn preprocess(&mut self, _key: &str, _options: GetOptions<'_>) -> Result<()> { + Ok(()) + } - /// Optionally update byte positions before they are passed to the other functions. - async fn update_byte_positions( + /// Postprocess data blocks before they are returned to the client. + async fn postprocess( &self, _key: &str, positions_options: BytesPositionOptions<'_>, @@ -223,370 +257,8 @@ impl UrlFormatter for htsget_config::storage::local::LocalStorage { } } -/// A DataBlock is either a range of bytes, or a data blob that gets transformed into a data uri. -#[derive(Debug, PartialEq, Eq)] -pub enum DataBlock { - Range(BytesPosition), - Data(Vec, Option), -} - -impl DataBlock { - /// Convert a vec of bytes positions to a vec of data blocks. Merges bytes positions. - pub fn from_bytes_positions(positions: Vec) -> Vec { - BytesPosition::merge_all(positions) - .into_iter() - .map(DataBlock::Range) - .collect() - } - - /// Update the classes of all blocks so that they all contain a class, or None. Does not merge - /// byte positions. - pub fn update_classes(blocks: Vec) -> Vec { - if blocks.iter().all(|block| match block { - DataBlock::Range(range) => range.class.is_some(), - DataBlock::Data(_, class) => class.is_some(), - }) { - blocks - } else { - blocks - .into_iter() - .map(|block| match block { - DataBlock::Range(range) => DataBlock::Range(range.set_class(None)), - DataBlock::Data(data, _) => DataBlock::Data(data, None), - }) - .collect() - } - } -} - -/// A byte position has an inclusive start value, and an exclusive end value. This is analogous to -/// query start and end parameters. The class represents the class type for this byte position when -/// formatted into url responses. The class is set to `Header` for byte positions containing only -/// header bytes, `Body` for byte positions containing only body bytes, and None for byte positions -/// with a mix of header and body bytes. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct BytesPosition { - start: Option, - end: Option, - class: Option, -} - -/// A bytes range has an inclusive start and end value. This is analogous to http bytes ranges. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct BytesRange { - start: Option, - end: Option, -} - -impl From<&BytesRange> for String { - fn from(ranges: &BytesRange) -> Self { - if ranges.start.is_none() && ranges.end.is_none() { - return "".to_string(); - } - ranges.to_string() - } -} - -impl Display for BytesRange { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let start = self - .start - .map(|start| start.to_string()) - .unwrap_or_else(|| "0".to_string()); - let end = self.end.map(|end| end.to_string()).unwrap_or_default(); - write!(f, "bytes={start}-{end}") - } -} - -/// Convert from a http range to a bytes position. -impl FromStr for BytesPosition { - type Err = StorageError; - - fn from_str(range: &str) -> Result { - let range = range.replacen("bytes=", "", 1); - - let split: Vec<&str> = range.splitn(2, '-').collect(); - if split.len() > 2 { - return Err(StorageError::InternalError( - "failed to split range".to_string(), - )); - } - - let parse_range = |range: Option<&str>| { - let range = range.unwrap_or_default(); - if range.is_empty() { - Ok::<_, Self::Err>(None) - } else { - Ok(Some(range.parse().map_err(|err: ParseIntError| { - StorageError::InternalError(err.to_string()) - })?)) - } - }; - - let start = parse_range(split.first().copied())?; - let end = parse_range(split.last().copied())?.map(|value| value + 1); - - Ok(Self::new(start, end, None)) - } -} - -impl From<&BytesPosition> for BytesRange { - fn from(pos: &BytesPosition) -> Self { - Self::new(pos.start, pos.end.map(|value| value - 1)) - } -} - -impl BytesRange { - pub fn new(start: Option, end: Option) -> Self { - Self { start, end } - } -} - -impl BytesPosition { - pub fn new(start: Option, end: Option, class: Option) -> Self { - Self { start, end, class } - } - - pub fn with_start(mut self, start: u64) -> Self { - self.start = Some(start); - self - } - - pub fn with_end(mut self, end: u64) -> Self { - self.end = Some(end); - self - } - - pub fn with_class(self, class: Class) -> Self { - self.set_class(Some(class)) - } - - pub fn set_class(mut self, class: Option) -> Self { - self.class = class; - self - } - - pub fn get_start(&self) -> Option { - self.start - } - - pub fn get_end(&self) -> Option { - self.end - } - - pub fn overlaps(&self, range: &BytesPosition) -> bool { - let cond1 = match (self.start.as_ref(), range.end.as_ref()) { - (None, None) | (None, Some(_)) | (Some(_), None) => true, - (Some(start), Some(end)) => end >= start, - }; - let cond2 = match (self.end.as_ref(), range.start.as_ref()) { - (None, None) | (None, Some(_)) | (Some(_), None) => true, - (Some(end), Some(start)) => end >= start, - }; - cond1 && cond2 - } - - /// Merges position with the current BytesPosition, assuming that the two positions overlap. - pub fn merge_with(&mut self, position: &BytesPosition) -> &Self { - let start = self.start; - let end = self.end; - - self.start = match (start.as_ref(), position.start.as_ref()) { - (None, None) | (None, Some(_)) | (Some(_), None) => None, - (Some(a), Some(b)) => Some(*a.min(b)), - }; - self.end = match (end.as_ref(), position.end.as_ref()) { - (None, None) | (None, Some(_)) | (Some(_), None) => None, - (Some(a), Some(b)) => Some(*a.max(b)), - }; - - self.class = match (self.class.as_ref(), position.class.as_ref()) { - (Some(Class::Header), Some(Class::Header)) => Some(Class::Header), - (Some(Class::Body), Some(Class::Body)) => Some(Class::Body), - (_, _) => None, - }; - - self - } - - /// Merge ranges, assuming ending byte ranges are exclusive. - #[instrument(level = "trace", ret)] - pub fn merge_all(mut ranges: Vec) -> Vec { - if ranges.len() < 2 { - ranges - } else { - ranges.sort_by(|a, b| { - let a_start = a.get_start().unwrap_or(0); - let b_start = b.get_start().unwrap_or(0); - let start_ord = a_start.cmp(&b_start); - if start_ord == Ordering::Equal { - let a_end = a.get_end().unwrap_or(u64::MAX); - let b_end = b.get_end().unwrap_or(u64::MAX); - b_end.cmp(&a_end) - } else { - start_ord - } - }); - - let mut optimized_ranges = Vec::with_capacity(ranges.len()); - - let mut current_range = ranges[0].clone(); - - for range in ranges.iter().skip(1) { - if current_range.overlaps(range) { - current_range.merge_with(range); - } else { - optimized_ranges.push(current_range); - current_range = range.clone(); - } - } - - optimized_ranges.push(current_range); - - optimized_ranges - } - } -} - -#[derive(Debug)] -pub struct GetOptions<'a> { - range: BytesPosition, - request_headers: &'a HeaderMap, -} - -impl<'a> GetOptions<'a> { - pub fn new(range: BytesPosition, request_headers: &'a HeaderMap) -> Self { - Self { - range, - request_headers, - } - } - - pub fn new_with_default_range(request_headers: &'a HeaderMap) -> Self { - Self::new(Default::default(), request_headers) - } - - pub fn with_max_length(mut self, max_length: u64) -> Self { - self.range = BytesPosition::default().with_start(0).with_end(max_length); - self - } - - pub fn with_range(mut self, range: BytesPosition) -> Self { - self.range = range; - self - } - - /// Get the range. - pub fn range(&self) -> &BytesPosition { - &self.range - } - - /// Get the request headers. - pub fn request_headers(&self) -> &'a HeaderMap { - self.request_headers - } -} - -#[derive(Debug, Clone)] -pub struct BytesPositionOptions<'a> { - positions: Vec, - headers: &'a HeaderMap, -} - -impl<'a> BytesPositionOptions<'a> { - pub fn new(positions: Vec, headers: &'a HeaderMap) -> Self { - Self { positions, headers } - } - - /// Get the response headers. - pub fn headers(&self) -> &'a HeaderMap { - self.headers - } - - pub fn positions(&self) -> &Vec { - &self.positions - } - - /// Get the inner value. - pub fn into_inner(self) -> Vec { - self.positions - } - - /// Merge all bytes positions - pub fn merge_all(mut self) -> Self { - self.positions = BytesPosition::merge_all(self.positions); - self - } -} - -#[derive(Debug)] -pub struct RangeUrlOptions<'a> { - range: BytesPosition, - response_headers: &'a HeaderMap, -} - -impl<'a> RangeUrlOptions<'a> { - pub fn new(range: BytesPosition, response_headers: &'a HeaderMap) -> Self { - Self { - range, - response_headers, - } - } - - pub fn new_with_default_range(request_headers: &'a HeaderMap) -> Self { - Self::new(Default::default(), request_headers) - } - - pub fn with_range(mut self, range: BytesPosition) -> Self { - self.range = range; - self - } - - pub fn apply(self, url: Url) -> Url { - let range: String = String::from(&BytesRange::from(self.range())); - - let url = if range.is_empty() { - url - } else { - url.add_headers(Headers::default().with_header("Range", range)) - }; - - url.set_class(self.range().class) - } - - /// Get the range. - pub fn range(&self) -> &BytesPosition { - &self.range - } - - /// Get the response headers. - pub fn response_headers(&self) -> &'a HeaderMap { - self.response_headers - } -} - -/// A struct to represent options passed to a `Storage` head call. -#[derive(Debug, Clone)] -pub struct HeadOptions<'a> { - request_headers: &'a HeaderMap, -} - -impl<'a> HeadOptions<'a> { - /// Create a new HeadOptions struct. - pub fn new(request_headers: &'a HeaderMap) -> Self { - Self { request_headers } - } - - /// Get the request headers. - pub fn request_headers(&self) -> &'a HeaderMap { - self.request_headers - } -} - #[cfg(test)] mod tests { - use std::collections::HashMap; - use http::uri::Authority; use crate::local::LocalStorage; @@ -595,339 +267,6 @@ mod tests { use super::*; - #[test] - fn bytes_range_overlapping_and_merge() { - let test_cases = vec![ - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(3), Some(5), None), - None, - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(3), None, None), - None, - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(2), Some(4), None), - Some(BytesPosition::new(None, Some(4), None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(2), None, None), - Some(BytesPosition::new(None, None, None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(1), Some(3), None), - Some(BytesPosition::new(None, Some(3), None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(1), None, None), - Some(BytesPosition::new(None, None, None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(0), Some(2), None), - Some(BytesPosition::new(None, Some(2), None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(None, Some(2), None), - Some(BytesPosition::new(None, Some(2), None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(0), Some(1), None), - Some(BytesPosition::new(None, Some(2), None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(None, Some(1), None), - Some(BytesPosition::new(None, Some(2), None)), - ), - ( - BytesPosition::new(None, Some(2), None), - BytesPosition::new(None, None, None), - Some(BytesPosition::new(None, None, None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(6), Some(8), None), - None, - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(6), None, None), - None, - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(4), Some(6), None), - Some(BytesPosition::new(Some(2), Some(6), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(4), None, None), - Some(BytesPosition::new(Some(2), None, None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(3), Some(5), None), - Some(BytesPosition::new(Some(2), Some(5), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(3), None, None), - Some(BytesPosition::new(Some(2), None, None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(2), Some(3), None), - Some(BytesPosition::new(Some(2), Some(4), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(None, Some(3), None), - Some(BytesPosition::new(None, Some(4), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(1), Some(3), None), - Some(BytesPosition::new(Some(1), Some(4), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(None, Some(3), None), - Some(BytesPosition::new(None, Some(4), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(0), Some(2), None), - Some(BytesPosition::new(Some(0), Some(4), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(None, Some(2), None), - Some(BytesPosition::new(None, Some(4), None)), - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(Some(0), Some(1), None), - None, - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(None, Some(1), None), - None, - ), - ( - BytesPosition::new(Some(2), Some(4), None), - BytesPosition::new(None, None, None), - Some(BytesPosition::new(None, None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(Some(4), Some(6), None), - Some(BytesPosition::new(Some(2), None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(Some(4), None, None), - Some(BytesPosition::new(Some(2), None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(Some(2), Some(4), None), - Some(BytesPosition::new(Some(2), None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(Some(2), None, None), - Some(BytesPosition::new(Some(2), None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(Some(1), Some(3), None), - Some(BytesPosition::new(Some(1), None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(None, Some(3), None), - Some(BytesPosition::new(None, None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(Some(0), Some(2), None), - Some(BytesPosition::new(Some(0), None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(None, Some(2), None), - Some(BytesPosition::new(None, None, None)), - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(Some(0), Some(1), None), - None, - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(None, Some(1), None), - None, - ), - ( - BytesPosition::new(Some(2), None, None), - BytesPosition::new(None, None, None), - Some(BytesPosition::new(None, None, None)), - ), - ( - BytesPosition::new(None, None, None), - BytesPosition::new(None, None, None), - Some(BytesPosition::new(None, None, None)), - ), - ]; - - for (index, (a, b, expected)) in test_cases.iter().enumerate() { - println!("Test case {index}"); - println!(" {a:?}"); - println!(" {b:?}"); - println!(" {expected:?}"); - - if a.overlaps(b) { - assert_eq!(*a.clone().merge_with(b), expected.clone().unwrap()); - } else { - assert!(expected.is_none()) - } - } - } - - #[test] - fn bytes_range_merge_all_when_list_is_empty() { - assert_eq!(BytesPosition::merge_all(Vec::new()), Vec::new()); - } - - #[test] - fn bytes_range_merge_all_when_list_has_one_range() { - assert_eq!( - BytesPosition::merge_all(vec![BytesPosition::default()]), - vec![BytesPosition::default()] - ); - } - - #[test] - fn bytes_position_merge_class_header() { - assert_eq!( - BytesPosition::merge_all(vec![ - BytesPosition::new(None, Some(1), Some(Class::Header)), - BytesPosition::new(None, Some(2), Some(Class::Header)) - ]), - vec![BytesPosition::new(None, Some(2), Some(Class::Header))] - ); - } - - #[test] - fn bytes_position_merge_class_body() { - assert_eq!( - BytesPosition::merge_all(vec![ - BytesPosition::new(None, Some(1), Some(Class::Body)), - BytesPosition::new(None, Some(3), Some(Class::Body)) - ]), - vec![BytesPosition::new(None, Some(3), Some(Class::Body))] - ); - } - - #[test] - fn bytes_position_merge_class_none() { - assert_eq!( - BytesPosition::merge_all(vec![ - BytesPosition::new(Some(1), Some(2), None), - BytesPosition::new(Some(2), Some(3), None) - ]), - vec![BytesPosition::new(Some(1), Some(3), None)] - ); - } - - #[test] - fn bytes_position_merge_class_different() { - assert_eq!( - BytesPosition::merge_all(vec![ - BytesPosition::new(Some(1), Some(2), Some(Class::Header)), - BytesPosition::new(Some(2), Some(3), Some(Class::Body)) - ]), - vec![BytesPosition::new(Some(1), Some(3), None)] - ); - } - - #[test] - fn bytes_range_merge_all_when_list_has_many_ranges() { - let ranges = vec![ - BytesPosition::new(None, Some(1), None), - BytesPosition::new(Some(1), Some(2), None), - BytesPosition::new(Some(5), Some(6), None), - BytesPosition::new(Some(5), Some(8), None), - BytesPosition::new(Some(6), Some(7), None), - BytesPosition::new(Some(4), Some(5), None), - BytesPosition::new(Some(3), Some(6), None), - BytesPosition::new(Some(10), Some(12), None), - BytesPosition::new(Some(10), Some(12), None), - BytesPosition::new(Some(10), Some(14), None), - BytesPosition::new(Some(14), Some(15), None), - BytesPosition::new(Some(12), Some(16), None), - BytesPosition::new(Some(17), Some(19), None), - BytesPosition::new(Some(21), Some(23), None), - BytesPosition::new(Some(18), Some(22), None), - BytesPosition::new(Some(24), None, None), - BytesPosition::new(Some(24), Some(30), None), - BytesPosition::new(Some(31), Some(33), None), - BytesPosition::new(Some(35), None, None), - ]; - - let expected_ranges = vec![ - BytesPosition::new(None, Some(2), None), - BytesPosition::new(Some(3), Some(8), None), - BytesPosition::new(Some(10), Some(16), None), - BytesPosition::new(Some(17), Some(23), None), - BytesPosition::new(Some(24), None, None), - ]; - - assert_eq!(BytesPosition::merge_all(ranges), expected_ranges); - } - - #[test] - fn bytes_position_new() { - let result = BytesPosition::new(Some(1), Some(2), Some(Class::Header)); - assert_eq!(result.start, Some(1)); - assert_eq!(result.end, Some(2)); - assert_eq!(result.class, Some(Class::Header)); - } - - #[test] - fn bytes_position_with_start() { - let result = BytesPosition::default().with_start(1); - assert_eq!(result.start, Some(1)); - } - - #[test] - fn bytes_position_with_end() { - let result = BytesPosition::default().with_end(1); - assert_eq!(result.end, Some(1)); - } - - #[test] - fn bytes_position_with_class() { - let result = BytesPosition::default().with_class(Class::Header); - assert_eq!(result.class, Some(Class::Header)); - } - - #[test] - fn bytes_position_set_class() { - let result = BytesPosition::default().set_class(Some(Class::Header)); - assert_eq!(result.class, Some(Class::Header)); - } - #[test] fn data_url() { let result = LocalStorage::::new( @@ -941,130 +280,6 @@ mod tests { assert_eq!(result, b"Hello World!"); } - #[test] - fn data_block_update_classes_all_some() { - let blocks = DataBlock::update_classes(vec![ - DataBlock::Range(BytesPosition::new(None, Some(1), Some(Class::Body))), - DataBlock::Data(vec![], Some(Class::Header)), - ]); - for block in blocks { - let class = match block { - DataBlock::Range(pos) => pos.class, - DataBlock::Data(_, class) => class, - }; - assert!(class.is_some()); - } - } - - #[test] - fn data_block_update_classes_one_none() { - let blocks = DataBlock::update_classes(vec![ - DataBlock::Range(BytesPosition::new(None, Some(1), Some(Class::Body))), - DataBlock::Data(vec![], None), - ]); - for block in blocks { - let class = match block { - DataBlock::Range(pos) => pos.class, - DataBlock::Data(_, class) => class, - }; - assert!(class.is_none()); - } - } - - #[test] - fn data_block_from_bytes_positions() { - let blocks = DataBlock::from_bytes_positions(vec![ - BytesPosition::new(None, Some(1), None), - BytesPosition::new(Some(1), Some(2), None), - ]); - assert_eq!( - blocks, - vec![DataBlock::Range(BytesPosition::new(None, Some(2), None))] - ); - } - - #[test] - fn byte_range_from_byte_position() { - let result: BytesRange = BytesRange::from(&BytesPosition::default().with_start(5).with_end(10)); - let expected = BytesRange::new(Some(5), Some(9)); - assert_eq!(result, expected); - } - - #[test] - fn get_options_with_max_length() { - let request_headers = Default::default(); - let result = GetOptions::new_with_default_range(&request_headers).with_max_length(1); - assert_eq!( - result.range(), - &BytesPosition::default().with_start(0).with_end(1) - ); - } - - #[test] - fn get_options_with_range() { - let request_headers = Default::default(); - let result = GetOptions::new_with_default_range(&request_headers) - .with_range(BytesPosition::new(Some(5), Some(11), Some(Class::Header))); - assert_eq!( - result.range(), - &BytesPosition::new(Some(5), Some(11), Some(Class::Header)) - ); - } - - #[test] - fn url_options_with_range() { - let request_headers = Default::default(); - let result = RangeUrlOptions::new_with_default_range(&request_headers) - .with_range(BytesPosition::new(Some(5), Some(11), Some(Class::Header))); - assert_eq!( - result.range(), - &BytesPosition::new(Some(5), Some(11), Some(Class::Header)) - ); - } - - #[test] - fn url_options_apply_with_bytes_range() { - let result = RangeUrlOptions::new( - BytesPosition::new(Some(5), Some(11), Some(Class::Header)), - &Default::default(), - ) - .apply(Url::new("")); - println!("{result:?}"); - assert_eq!( - result, - Url::new("") - .with_headers(Headers::new(HashMap::new()).with_header("Range", "bytes=5-10")) - .with_class(Class::Header) - ); - } - - #[test] - fn url_options_apply_no_bytes_range() { - let result = RangeUrlOptions::new_with_default_range(&Default::default()).apply(Url::new("")); - assert_eq!(result, Url::new("")); - } - - #[test] - fn url_options_apply_with_headers() { - let result = RangeUrlOptions::new( - BytesPosition::new(Some(5), Some(11), Some(Class::Header)), - &Default::default(), - ) - .apply(Url::new("").with_headers(Headers::default().with_header("header", "value"))); - println!("{result:?}"); - - assert_eq!( - result, - Url::new("") - .with_headers( - Headers::new(HashMap::new()) - .with_header("Range", "bytes=5-10") - .with_header("header", "value") - ) - .with_class(Class::Header) - ); - } - #[test] fn http_formatter_authority() { let formatter = ConfigLocalStorage::new( @@ -1089,18 +304,6 @@ mod tests { test_formatter_authority(formatter, "https"); } - #[test] - fn htsget_error_from_storage_not_found() { - let result = HtsGetError::from(StorageError::KeyNotFound("error".to_string())); - assert!(matches!(result, HtsGetError::NotFound(_))); - } - - #[test] - fn htsget_error_from_storage_invalid_key() { - let result = HtsGetError::from(StorageError::InvalidKey("error".to_string())); - assert!(matches!(result, HtsGetError::NotFound(_))); - } - fn test_formatter_authority(formatter: ConfigLocalStorage, scheme: &str) { assert_eq!( formatter.format_url("path").unwrap(), diff --git a/htsget-storage/src/local.rs b/htsget-storage/src/local.rs index 7f5dd3a1..60cb232b 100644 --- a/htsget-storage/src/local.rs +++ b/htsget-storage/src/local.rs @@ -5,7 +5,7 @@ use std::fmt::Debug; use std::io::ErrorKind; use std::path::{Path, PathBuf}; -use crate::{HeadOptions, StorageTrait, UrlFormatter}; +use crate::{HeadOptions, StorageMiddleware, StorageTrait, UrlFormatter}; use crate::{Streamable, Url as HtsGetUrl}; use async_trait::async_trait; use tokio::fs; @@ -78,7 +78,10 @@ impl LocalStorage { } #[async_trait] -impl StorageTrait for LocalStorage { +impl StorageMiddleware for LocalStorage {} + +#[async_trait] +impl StorageTrait for LocalStorage { /// Get the file at the location of the key. #[instrument(level = "debug", skip(self))] async fn get(&self, key: &str, _options: GetOptions<'_>) -> Result { @@ -140,10 +143,10 @@ pub(crate) mod tests { use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; use htsget_config::types::Scheme; - use crate::{BytesPosition, GetOptions, RangeUrlOptions, StorageError}; - use crate::{Headers, Url}; - use super::*; + use crate::types::BytesPosition; + use crate::{GetOptions, RangeUrlOptions, StorageError}; + use crate::{Headers, Url}; #[tokio::test] async fn get_non_existing_key() { diff --git a/htsget-storage/src/s3.rs b/htsget-storage/src/s3.rs index ec59bc14..5fd9bae7 100644 --- a/htsget-storage/src/s3.rs +++ b/htsget-storage/src/s3.rs @@ -25,14 +25,13 @@ use tokio_util::io::StreamReader; use tracing::instrument; use tracing::{debug, warn}; +use super::{GetOptions, RangeUrlOptions, Result}; use crate::s3::Retrieval::{Delayed, Immediate}; +use crate::types::{BytesPosition, BytesRange}; use crate::StorageError::{AwsS3Error, IoError, KeyNotFound}; -use crate::{BytesPosition, HeadOptions, StorageError}; -use crate::{BytesRange, StorageTrait}; +use crate::{HeadOptions, StorageError, StorageMiddleware, StorageTrait}; use crate::{Streamable, Url}; -use super::{GetOptions, RangeUrlOptions, Result}; - /// Represents data classes that can be retrieved immediately or after a delay. /// Specifically, Glacier Flexible, Glacier Deep Archive, and Intelligent Tiering archive /// tiers have delayed retrieval, unless they have been restored. @@ -242,6 +241,9 @@ impl Stream for S3Stream { } } +#[async_trait] +impl StorageMiddleware for S3Storage {} + #[async_trait] impl StorageTrait for S3Storage { /// Gets the actual s3 object as a buffered reader. @@ -296,8 +298,9 @@ pub(crate) mod tests { use crate::local::tests::create_local_test_files; use crate::s3::S3Storage; + use crate::types::BytesPosition; use crate::Headers; - use crate::{BytesPosition, GetOptions, RangeUrlOptions, StorageTrait}; + use crate::{GetOptions, RangeUrlOptions, StorageTrait}; use crate::{HeadOptions, StorageError}; pub(crate) async fn with_aws_s3_storage_fn(test: F, folder_name: String, base_path: &Path) diff --git a/htsget-storage/src/types.rs b/htsget-storage/src/types.rs new file mode 100644 index 00000000..b1e8e8e4 --- /dev/null +++ b/htsget-storage/src/types.rs @@ -0,0 +1,804 @@ +use htsget_config::types::{Class, Headers, Url}; +use http::HeaderMap; +use std::cmp::Ordering; +use std::fmt; +use std::fmt::{Display, Formatter}; +use tracing::instrument; + +/// A DataBlock is either a range of bytes, or a data blob that gets transformed into a data uri. +#[derive(Debug, PartialEq, Eq)] +pub enum DataBlock { + Range(BytesPosition), + Data(Vec, Option), +} + +impl DataBlock { + /// Convert a vec of bytes positions to a vec of data blocks. Merges bytes positions. + pub fn from_bytes_positions(positions: Vec) -> Vec { + BytesPosition::merge_all(positions) + .into_iter() + .map(DataBlock::Range) + .collect() + } + + /// Update the classes of all blocks so that they all contain a class, or None. Does not merge + /// byte positions. + pub fn update_classes(blocks: Vec) -> Vec { + if blocks.iter().all(|block| match block { + DataBlock::Range(range) => range.class.is_some(), + DataBlock::Data(_, class) => class.is_some(), + }) { + blocks + } else { + blocks + .into_iter() + .map(|block| match block { + DataBlock::Range(range) => DataBlock::Range(range.set_class(None)), + DataBlock::Data(data, _) => DataBlock::Data(data, None), + }) + .collect() + } + } +} + +/// A byte position has an inclusive start value, and an exclusive end value. This is analogous to +/// query start and end parameters. The class represents the class type for this byte position when +/// formatted into url responses. The class is set to `Header` for byte positions containing only +/// header bytes, `Body` for byte positions containing only body bytes, and None for byte positions +/// with a mix of header and body bytes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BytesPosition { + pub(crate) start: Option, + pub(crate) end: Option, + pub(crate) class: Option, +} + +/// A bytes range has an inclusive start and end value. This is analogous to http bytes ranges. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BytesRange { + start: Option, + end: Option, +} + +impl From<&BytesRange> for String { + fn from(ranges: &BytesRange) -> Self { + if ranges.start.is_none() && ranges.end.is_none() { + return "".to_string(); + } + ranges.to_string() + } +} + +impl Display for BytesRange { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let start = self + .start + .map(|start| start.to_string()) + .unwrap_or_else(|| "0".to_string()); + let end = self.end.map(|end| end.to_string()).unwrap_or_default(); + write!(f, "bytes={start}-{end}") + } +} + +impl From<&BytesPosition> for BytesRange { + fn from(pos: &BytesPosition) -> Self { + Self::new(pos.start, pos.end.map(|value| value - 1)) + } +} + +impl BytesRange { + pub fn new(start: Option, end: Option) -> Self { + Self { start, end } + } +} + +impl BytesPosition { + pub fn new(start: Option, end: Option, class: Option) -> Self { + Self { start, end, class } + } + + pub fn with_start(mut self, start: u64) -> Self { + self.start = Some(start); + self + } + + pub fn with_end(mut self, end: u64) -> Self { + self.end = Some(end); + self + } + + pub fn with_class(self, class: Class) -> Self { + self.set_class(Some(class)) + } + + pub fn set_class(mut self, class: Option) -> Self { + self.class = class; + self + } + + pub fn get_start(&self) -> Option { + self.start + } + + pub fn get_end(&self) -> Option { + self.end + } + + pub fn overlaps(&self, range: &BytesPosition) -> bool { + let cond1 = match (self.start.as_ref(), range.end.as_ref()) { + (None, None) | (None, Some(_)) | (Some(_), None) => true, + (Some(start), Some(end)) => end >= start, + }; + let cond2 = match (self.end.as_ref(), range.start.as_ref()) { + (None, None) | (None, Some(_)) | (Some(_), None) => true, + (Some(end), Some(start)) => end >= start, + }; + cond1 && cond2 + } + + /// Merges position with the current BytesPosition, assuming that the two positions overlap. + pub fn merge_with(&mut self, position: &BytesPosition) -> &Self { + let start = self.start; + let end = self.end; + + self.start = match (start.as_ref(), position.start.as_ref()) { + (None, None) | (None, Some(_)) | (Some(_), None) => None, + (Some(a), Some(b)) => Some(*a.min(b)), + }; + self.end = match (end.as_ref(), position.end.as_ref()) { + (None, None) | (None, Some(_)) | (Some(_), None) => None, + (Some(a), Some(b)) => Some(*a.max(b)), + }; + + self.class = match (self.class.as_ref(), position.class.as_ref()) { + (Some(Class::Header), Some(Class::Header)) => Some(Class::Header), + (Some(Class::Body), Some(Class::Body)) => Some(Class::Body), + (_, _) => None, + }; + + self + } + + /// Merge ranges, assuming ending byte ranges are exclusive. + #[instrument(level = "trace", ret)] + pub fn merge_all(mut ranges: Vec) -> Vec { + if ranges.len() < 2 { + ranges + } else { + ranges.sort_by(|a, b| { + let a_start = a.get_start().unwrap_or(0); + let b_start = b.get_start().unwrap_or(0); + let start_ord = a_start.cmp(&b_start); + if start_ord == Ordering::Equal { + let a_end = a.get_end().unwrap_or(u64::MAX); + let b_end = b.get_end().unwrap_or(u64::MAX); + b_end.cmp(&a_end) + } else { + start_ord + } + }); + + let mut optimized_ranges = Vec::with_capacity(ranges.len()); + + let mut current_range = ranges[0].clone(); + + for range in ranges.iter().skip(1) { + if current_range.overlaps(range) { + current_range.merge_with(range); + } else { + optimized_ranges.push(current_range); + current_range = range.clone(); + } + } + + optimized_ranges.push(current_range); + + optimized_ranges + } + } +} + +#[derive(Debug, Clone)] +pub struct GetOptions<'a> { + pub(crate) range: BytesPosition, + pub(crate) request_headers: &'a HeaderMap, +} + +impl<'a> GetOptions<'a> { + pub fn new(range: BytesPosition, request_headers: &'a HeaderMap) -> Self { + Self { + range, + request_headers, + } + } + + pub fn new_with_default_range(request_headers: &'a HeaderMap) -> Self { + Self::new(Default::default(), request_headers) + } + + pub fn with_max_length(mut self, max_length: u64) -> Self { + self.range = BytesPosition::default().with_start(0).with_end(max_length); + self + } + + pub fn with_range(mut self, range: BytesPosition) -> Self { + self.range = range; + self + } + + /// Get the range. + pub fn range(&self) -> &BytesPosition { + &self.range + } + + /// Get the request headers. + pub fn request_headers(&self) -> &'a HeaderMap { + self.request_headers + } +} + +#[derive(Debug, Clone)] +pub struct BytesPositionOptions<'a> { + pub(crate) positions: Vec, + pub(crate) headers: &'a HeaderMap, +} + +impl<'a> BytesPositionOptions<'a> { + pub fn new(positions: Vec, headers: &'a HeaderMap) -> Self { + Self { positions, headers } + } + + /// Get the response headers. + pub fn headers(&self) -> &'a HeaderMap { + self.headers + } + + pub fn positions(&self) -> &Vec { + &self.positions + } + + /// Get the inner value. + pub fn into_inner(self) -> Vec { + self.positions + } + + /// Merge all bytes positions + pub fn merge_all(mut self) -> Self { + self.positions = BytesPosition::merge_all(self.positions); + self + } +} + +#[derive(Debug, Clone)] +pub struct RangeUrlOptions<'a> { + range: BytesPosition, + response_headers: &'a HeaderMap, +} + +impl<'a> RangeUrlOptions<'a> { + pub fn new(range: BytesPosition, response_headers: &'a HeaderMap) -> Self { + Self { + range, + response_headers, + } + } + + pub fn new_with_default_range(request_headers: &'a HeaderMap) -> Self { + Self::new(Default::default(), request_headers) + } + + pub fn with_range(mut self, range: BytesPosition) -> Self { + self.range = range; + self + } + + pub fn apply(self, url: Url) -> Url { + let range: String = String::from(&BytesRange::from(self.range())); + + let url = if range.is_empty() { + url + } else { + url.add_headers(Headers::default().with_header("Range", range)) + }; + + url.set_class(self.range().class) + } + + /// Get the range. + pub fn range(&self) -> &BytesPosition { + &self.range + } + + /// Get the response headers. + pub fn response_headers(&self) -> &'a HeaderMap { + self.response_headers + } +} + +/// A struct to represent options passed to a `Storage` head call. +#[derive(Debug, Clone)] +pub struct HeadOptions<'a> { + request_headers: &'a HeaderMap, +} + +impl<'a> HeadOptions<'a> { + /// Create a new HeadOptions struct. + pub fn new(request_headers: &'a HeaderMap) -> Self { + Self { request_headers } + } + + /// Get the request headers. + pub fn request_headers(&self) -> &'a HeaderMap { + self.request_headers + } +} + +impl<'a> From<&'a GetOptions<'a>> for HeadOptions<'a> { + fn from(options: &'a GetOptions<'a>) -> Self { + Self::new(options.request_headers()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + #[test] + fn bytes_range_overlapping_and_merge() { + let test_cases = vec![ + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(3), Some(5), None), + None, + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(3), None, None), + None, + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(2), Some(4), None), + Some(BytesPosition::new(None, Some(4), None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(2), None, None), + Some(BytesPosition::new(None, None, None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(1), Some(3), None), + Some(BytesPosition::new(None, Some(3), None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(1), None, None), + Some(BytesPosition::new(None, None, None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(0), Some(2), None), + Some(BytesPosition::new(None, Some(2), None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(None, Some(2), None), + Some(BytesPosition::new(None, Some(2), None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(0), Some(1), None), + Some(BytesPosition::new(None, Some(2), None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(None, Some(1), None), + Some(BytesPosition::new(None, Some(2), None)), + ), + ( + BytesPosition::new(None, Some(2), None), + BytesPosition::new(None, None, None), + Some(BytesPosition::new(None, None, None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(6), Some(8), None), + None, + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(6), None, None), + None, + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(4), Some(6), None), + Some(BytesPosition::new(Some(2), Some(6), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(4), None, None), + Some(BytesPosition::new(Some(2), None, None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(3), Some(5), None), + Some(BytesPosition::new(Some(2), Some(5), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(3), None, None), + Some(BytesPosition::new(Some(2), None, None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(2), Some(3), None), + Some(BytesPosition::new(Some(2), Some(4), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(None, Some(3), None), + Some(BytesPosition::new(None, Some(4), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(1), Some(3), None), + Some(BytesPosition::new(Some(1), Some(4), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(None, Some(3), None), + Some(BytesPosition::new(None, Some(4), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(0), Some(2), None), + Some(BytesPosition::new(Some(0), Some(4), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(None, Some(2), None), + Some(BytesPosition::new(None, Some(4), None)), + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(Some(0), Some(1), None), + None, + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(None, Some(1), None), + None, + ), + ( + BytesPosition::new(Some(2), Some(4), None), + BytesPosition::new(None, None, None), + Some(BytesPosition::new(None, None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(Some(4), Some(6), None), + Some(BytesPosition::new(Some(2), None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(Some(4), None, None), + Some(BytesPosition::new(Some(2), None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(Some(2), Some(4), None), + Some(BytesPosition::new(Some(2), None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(Some(2), None, None), + Some(BytesPosition::new(Some(2), None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(Some(1), Some(3), None), + Some(BytesPosition::new(Some(1), None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(None, Some(3), None), + Some(BytesPosition::new(None, None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(Some(0), Some(2), None), + Some(BytesPosition::new(Some(0), None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(None, Some(2), None), + Some(BytesPosition::new(None, None, None)), + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(Some(0), Some(1), None), + None, + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(None, Some(1), None), + None, + ), + ( + BytesPosition::new(Some(2), None, None), + BytesPosition::new(None, None, None), + Some(BytesPosition::new(None, None, None)), + ), + ( + BytesPosition::new(None, None, None), + BytesPosition::new(None, None, None), + Some(BytesPosition::new(None, None, None)), + ), + ]; + + for (index, (a, b, expected)) in test_cases.iter().enumerate() { + println!("Test case {index}"); + println!(" {a:?}"); + println!(" {b:?}"); + println!(" {expected:?}"); + + if a.overlaps(b) { + assert_eq!(*a.clone().merge_with(b), expected.clone().unwrap()); + } else { + assert!(expected.is_none()) + } + } + } + + #[test] + fn bytes_range_merge_all_when_list_is_empty() { + assert_eq!(BytesPosition::merge_all(Vec::new()), Vec::new()); + } + + #[test] + fn bytes_range_merge_all_when_list_has_one_range() { + assert_eq!( + BytesPosition::merge_all(vec![BytesPosition::default()]), + vec![BytesPosition::default()] + ); + } + + #[test] + fn bytes_position_merge_class_header() { + assert_eq!( + BytesPosition::merge_all(vec![ + BytesPosition::new(None, Some(1), Some(Class::Header)), + BytesPosition::new(None, Some(2), Some(Class::Header)) + ]), + vec![BytesPosition::new(None, Some(2), Some(Class::Header))] + ); + } + + #[test] + fn bytes_position_merge_class_body() { + assert_eq!( + BytesPosition::merge_all(vec![ + BytesPosition::new(None, Some(1), Some(Class::Body)), + BytesPosition::new(None, Some(3), Some(Class::Body)) + ]), + vec![BytesPosition::new(None, Some(3), Some(Class::Body))] + ); + } + + #[test] + fn bytes_position_merge_class_none() { + assert_eq!( + BytesPosition::merge_all(vec![ + BytesPosition::new(Some(1), Some(2), None), + BytesPosition::new(Some(2), Some(3), None) + ]), + vec![BytesPosition::new(Some(1), Some(3), None)] + ); + } + + #[test] + fn bytes_position_merge_class_different() { + assert_eq!( + BytesPosition::merge_all(vec![ + BytesPosition::new(Some(1), Some(2), Some(Class::Header)), + BytesPosition::new(Some(2), Some(3), Some(Class::Body)) + ]), + vec![BytesPosition::new(Some(1), Some(3), None)] + ); + } + + #[test] + fn bytes_range_merge_all_when_list_has_many_ranges() { + let ranges = vec![ + BytesPosition::new(None, Some(1), None), + BytesPosition::new(Some(1), Some(2), None), + BytesPosition::new(Some(5), Some(6), None), + BytesPosition::new(Some(5), Some(8), None), + BytesPosition::new(Some(6), Some(7), None), + BytesPosition::new(Some(4), Some(5), None), + BytesPosition::new(Some(3), Some(6), None), + BytesPosition::new(Some(10), Some(12), None), + BytesPosition::new(Some(10), Some(12), None), + BytesPosition::new(Some(10), Some(14), None), + BytesPosition::new(Some(14), Some(15), None), + BytesPosition::new(Some(12), Some(16), None), + BytesPosition::new(Some(17), Some(19), None), + BytesPosition::new(Some(21), Some(23), None), + BytesPosition::new(Some(18), Some(22), None), + BytesPosition::new(Some(24), None, None), + BytesPosition::new(Some(24), Some(30), None), + BytesPosition::new(Some(31), Some(33), None), + BytesPosition::new(Some(35), None, None), + ]; + + let expected_ranges = vec![ + BytesPosition::new(None, Some(2), None), + BytesPosition::new(Some(3), Some(8), None), + BytesPosition::new(Some(10), Some(16), None), + BytesPosition::new(Some(17), Some(23), None), + BytesPosition::new(Some(24), None, None), + ]; + + assert_eq!(BytesPosition::merge_all(ranges), expected_ranges); + } + + #[test] + fn bytes_position_new() { + let result = BytesPosition::new(Some(1), Some(2), Some(Class::Header)); + assert_eq!(result.start, Some(1)); + assert_eq!(result.end, Some(2)); + assert_eq!(result.class, Some(Class::Header)); + } + + #[test] + fn bytes_position_with_start() { + let result = BytesPosition::default().with_start(1); + assert_eq!(result.start, Some(1)); + } + + #[test] + fn bytes_position_with_end() { + let result = BytesPosition::default().with_end(1); + assert_eq!(result.end, Some(1)); + } + + #[test] + fn bytes_position_with_class() { + let result = BytesPosition::default().with_class(Class::Header); + assert_eq!(result.class, Some(Class::Header)); + } + + #[test] + fn bytes_position_set_class() { + let result = BytesPosition::default().set_class(Some(Class::Header)); + assert_eq!(result.class, Some(Class::Header)); + } + + #[test] + fn data_block_update_classes_all_some() { + let blocks = DataBlock::update_classes(vec![ + DataBlock::Range(BytesPosition::new(None, Some(1), Some(Class::Body))), + DataBlock::Data(vec![], Some(Class::Header)), + ]); + for block in blocks { + let class = match block { + DataBlock::Range(pos) => pos.class, + DataBlock::Data(_, class) => class, + }; + assert!(class.is_some()); + } + } + + #[test] + fn data_block_update_classes_one_none() { + let blocks = DataBlock::update_classes(vec![ + DataBlock::Range(BytesPosition::new(None, Some(1), Some(Class::Body))), + DataBlock::Data(vec![], None), + ]); + for block in blocks { + let class = match block { + DataBlock::Range(pos) => pos.class, + DataBlock::Data(_, class) => class, + }; + assert!(class.is_none()); + } + } + + #[test] + fn data_block_from_bytes_positions() { + let blocks = DataBlock::from_bytes_positions(vec![ + BytesPosition::new(None, Some(1), None), + BytesPosition::new(Some(1), Some(2), None), + ]); + assert_eq!( + blocks, + vec![DataBlock::Range(BytesPosition::new(None, Some(2), None))] + ); + } + + #[test] + fn byte_range_from_byte_position() { + let result: BytesRange = BytesRange::from(&BytesPosition::default().with_start(5).with_end(10)); + let expected = BytesRange::new(Some(5), Some(9)); + assert_eq!(result, expected); + } + + #[test] + fn get_options_with_max_length() { + let request_headers = Default::default(); + let result = GetOptions::new_with_default_range(&request_headers).with_max_length(1); + assert_eq!( + result.range(), + &BytesPosition::default().with_start(0).with_end(1) + ); + } + + #[test] + fn get_options_with_range() { + let request_headers = Default::default(); + let result = GetOptions::new_with_default_range(&request_headers) + .with_range(BytesPosition::new(Some(5), Some(11), Some(Class::Header))); + assert_eq!( + result.range(), + &BytesPosition::new(Some(5), Some(11), Some(Class::Header)) + ); + } + + #[test] + fn url_options_with_range() { + let request_headers = Default::default(); + let result = RangeUrlOptions::new_with_default_range(&request_headers) + .with_range(BytesPosition::new(Some(5), Some(11), Some(Class::Header))); + assert_eq!( + result.range(), + &BytesPosition::new(Some(5), Some(11), Some(Class::Header)) + ); + } + + #[test] + fn url_options_apply_with_bytes_range() { + let result = RangeUrlOptions::new( + BytesPosition::new(Some(5), Some(11), Some(Class::Header)), + &Default::default(), + ) + .apply(Url::new("")); + println!("{result:?}"); + assert_eq!( + result, + Url::new("") + .with_headers(Headers::new(HashMap::new()).with_header("Range", "bytes=5-10")) + .with_class(Class::Header) + ); + } + + #[test] + fn url_options_apply_no_bytes_range() { + let result = RangeUrlOptions::new_with_default_range(&Default::default()).apply(Url::new("")); + assert_eq!(result, Url::new("")); + } + + #[test] + fn url_options_apply_with_headers() { + let result = RangeUrlOptions::new( + BytesPosition::new(Some(5), Some(11), Some(Class::Header)), + &Default::default(), + ) + .apply(Url::new("").with_headers(Headers::default().with_header("header", "value"))); + println!("{result:?}"); + + assert_eq!( + result, + Url::new("") + .with_headers( + Headers::new(HashMap::new()) + .with_header("Range", "bytes=5-10") + .with_header("header", "value") + ) + .with_class(Class::Header) + ); + } +} diff --git a/htsget-storage/src/url.rs b/htsget-storage/src/url.rs index e6efc86e..10e9f9d0 100644 --- a/htsget-storage/src/url.rs +++ b/htsget-storage/src/url.rs @@ -16,7 +16,9 @@ use tracing::{debug, instrument}; use htsget_config::error; use crate::StorageError::{InternalError, KeyNotFound, ResponseError, UrlParseError}; -use crate::{GetOptions, HeadOptions, RangeUrlOptions, Result, StorageError, StorageTrait}; +use crate::{ + GetOptions, HeadOptions, RangeUrlOptions, Result, StorageError, StorageMiddleware, StorageTrait, +}; use crate::{Streamable, Url as HtsGetUrl}; /// A storage struct which derives data from HTTP URLs. @@ -191,6 +193,9 @@ impl Stream for UrlStream { } } +#[async_trait] +impl StorageMiddleware for UrlStorage {} + #[async_trait] impl StorageTrait for UrlStorage { #[instrument(level = "trace", skip(self))] diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index fbe5fe4c..3d518451 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -36,14 +36,14 @@ aws-mocks = [ ] s3-storage = ["htsget-config?/s3-storage"] url-storage = ["htsget-config?/url-storage"] -c4gh-experimental = ["dep:crypt4gh", "dep:htsget-config", "htsget-config/c4gh-experimental"] +experimental = ["dep:crypt4gh", "dep:htsget-config", "htsget-config/experimental"] default = [] [dependencies] # Server tests dependencies htsget-config = { version = "0.10.1", path = "../htsget-config", default-features = false, optional = true } -noodles = { version = "0.80", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } +noodles = { version = "0.82", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } tokio = { version = "1", features = ["rt-multi-thread", "fs"], optional = true } diff --git a/htsget-test/README.md b/htsget-test/README.md index eb25ab53..e7af1ff8 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -40,7 +40,7 @@ This crate has the following features: * `aws-mocks`: used to enable AWS mocking for tests. * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `c4gh-experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable `C4GHStorage` functionality. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies diff --git a/htsget-test/src/c4gh.rs b/htsget-test/src/c4gh.rs index 5092c40e..8f1de6e5 100644 --- a/htsget-test/src/c4gh.rs +++ b/htsget-test/src/c4gh.rs @@ -1,7 +1,8 @@ use crate::util::default_dir; -use crypt4gh::keys::get_private_key; -use crypt4gh::{decrypt, Keys}; +use crypt4gh::keys::{get_private_key, get_public_key}; +use crypt4gh::{decrypt, encrypt, Keys}; use htsget_config::storage::object::c4gh::{C4GHKeys, C4GHPath}; +use std::collections::HashSet; use std::io::{BufReader, BufWriter, Cursor}; pub fn decrypt_data(data: &[u8]) -> Vec { @@ -31,6 +32,33 @@ pub fn decrypt_data(data: &[u8]) -> Vec { writer.into_inner().unwrap().into_inner() } +pub fn encrypt_data(data: &[u8]) -> Vec { + let keys = get_private_key( + default_dir().join("data/c4gh/keys/alice.sec"), + Ok("".to_string()), + ) + .unwrap(); + let recipient_key = get_public_key(default_dir().join("data/c4gh/keys/bob.pub")).unwrap(); + + let mut reader = BufReader::new(Cursor::new(data)); + let mut writer = BufWriter::new(Cursor::new(vec![])); + + encrypt( + &HashSet::from_iter(vec![Keys { + method: 0, + privkey: keys, + recipient_pubkey: recipient_key, + }]), + &mut reader, + &mut writer, + 0, + None, + ) + .unwrap(); + + writer.into_inner().unwrap().into_inner() +} + pub fn get_decryption_keys() -> Vec { let private_key = default_dir().join("data/c4gh/keys/bob.sec"); let public_key = default_dir().join("data/c4gh/keys/alice.pub"); diff --git a/htsget-test/src/lib.rs b/htsget-test/src/lib.rs index 9fa40f2b..d5c10391 100644 --- a/htsget-test/src/lib.rs +++ b/htsget-test/src/lib.rs @@ -6,7 +6,7 @@ pub use htsget_config::{ #[cfg(feature = "aws-mocks")] pub mod aws_mocks; -#[cfg(feature = "c4gh-experimental")] +#[cfg(feature = "experimental")] pub mod c4gh; pub mod error; #[cfg(feature = "http")]