diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e5a58b7b..4544306ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,6 +123,9 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Test pixi run: pixi run test-slow + env: + PIXI_TEST_R2_ACCESS_KEY_ID: ${{ secrets.PIXI_TEST_R2_ACCESS_KEY_ID }} + PIXI_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.PIXI_TEST_R2_SECRET_ACCESS_KEY }} cargo-test-macos-aarch64: name: "cargo test | macos aarch64" @@ -142,6 +145,9 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Test pixi run: pixi run test-slow + env: + PIXI_TEST_R2_ACCESS_KEY_ID: ${{ secrets.PIXI_TEST_R2_ACCESS_KEY_ID }} + PIXI_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.PIXI_TEST_R2_SECRET_ACCESS_KEY }} cargo-test-macos-x86_64: name: "cargo test | macos x86_64" @@ -161,6 +167,9 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Test pixi run: pixi run test-slow + env: + PIXI_TEST_R2_ACCESS_KEY_ID: ${{ secrets.PIXI_TEST_R2_ACCESS_KEY_ID }} + PIXI_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.PIXI_TEST_R2_SECRET_ACCESS_KEY }} cargo-test-windows: name: "cargo test | windows" @@ -182,6 +191,9 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Test pixi run: pixi run test-slow + env: + PIXI_TEST_R2_ACCESS_KEY_ID: ${{ secrets.PIXI_TEST_R2_ACCESS_KEY_ID }} + PIXI_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.PIXI_TEST_R2_SECRET_ACCESS_KEY }} # # Builds the binary artifacts on different platforms diff --git a/Cargo.lock b/Cargo.lock index b7f4243e0..54156bd4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,9 +170,9 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "astral-tokio-tar" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9807072104b1b31db50a77aae7dfdb576a193ff85436c5ded97f601a608f17aa" +checksum = "65152cbda42e8ab5ecff69e8811e8333d69188c7d5c41e3eedb8d127e3f23b27" dependencies = [ "filetime", "futures-core", @@ -412,6 +412,381 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc47e70fc35d054c8fcd296d47a61711f043ac80534a10b4f741904f81e73a90" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7ce6d85596c4bcb3aba8ad5bb134b08e204c8a475c9999c1af9290f80aa8ad" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c54bab121fe1881a74c338c5f723d1592bf3b53167f80268a1274f404e1acc38" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c8234fd024f7ac61c4e44ea008029bde934250f371efe7d4a39708397b1080c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba60e1d519d6f23a9df712c04fdeadd7872ac911c84b2f62a8bda92e129b7962" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.2.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f45a1c384d7a393026bc5f5c177105aa9fa68e4749653b985707ac27d77295" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper 0.14.32", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.2.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.2.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backon" version = "1.3.0" @@ -447,6 +822,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.21.7" @@ -459,6 +840,22 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bincode" version = "1.3.3" @@ -596,6 +993,16 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -692,6 +1099,25 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap 2.7.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.2.10" @@ -870,6 +1296,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const_format" version = "0.2.34" @@ -925,6 +1357,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -934,6 +1390,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "crossbeam-channel" version = "0.5.14" @@ -968,6 +1434,28 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1111,6 +1599,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -1206,12 +1704,44 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "elsa" version = "1.10.0" @@ -1383,6 +1913,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "file_url" version = "0.2.2" @@ -1704,7 +2244,7 @@ dependencies = [ "futures-core", "futures-sink", "gloo-utils", - "http", + "http 1.2.0", "js-sys", "pin-project", "serde", @@ -1805,6 +2345,36 @@ dependencies = [ "async-trait", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.7.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.7" @@ -1816,7 +2386,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.2.0", "indexmap 2.7.0", "slab", "tokio", @@ -1930,6 +2500,17 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.2.0" @@ -1941,6 +2522,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1948,7 +2540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.2.0", ] [[package]] @@ -1959,8 +2551,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1973,7 +2565,7 @@ dependencies = [ "async-trait", "bincode", "cacache", - "http", + "http 1.2.0", "http-cache-semantics", "httpdate", "serde", @@ -1988,7 +2580,7 @@ checksum = "735586904a5ce0c13877c57cb4eb8195eb7c11ec1ffd64d4db053fb8559ca62e" dependencies = [ "anyhow", "async-trait", - "http", + "http 1.2.0", "http-cache", "http-cache-semantics", "reqwest", @@ -2003,7 +2595,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92baf25cf0b8c9246baecf3a444546360a97b569168fdf92563ee6a47829920c" dependencies = [ - "http", + "http 1.2.0", "http-serde", "reqwest", "serde", @@ -2022,7 +2614,7 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ - "http", + "http 1.2.0", "serde", ] @@ -2059,6 +2651,30 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.5.2" @@ -2068,9 +2684,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", "httparse", "itoa", "pin-project-lite", @@ -2079,6 +2695,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.5" @@ -2086,15 +2718,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.2.0", + "hyper 1.5.2", "hyper-util", "log", - "rustls", + "rustls 0.23.21", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.1", "tower-service", "webpki-roots", ] @@ -2107,7 +2739,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.5.2", "hyper-util", "native-tls", "tokio", @@ -2124,9 +2756,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -2580,16 +3212,16 @@ dependencies = [ "futures-channel", "futures-util", "gloo-net", - "http", + "http 1.2.0", "jsonrpsee-core", "pin-project", - "rustls", + "rustls 0.23.21", "rustls-pki-types", "rustls-platform-verifier", "soketto", "thiserror 1.0.69", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.1", "tokio-util", "tracing", "url", @@ -2605,8 +3237,8 @@ dependencies = [ "bytes", "futures-timer", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "http-body-util", "jsonrpsee-types", "pin-project", @@ -2628,13 +3260,13 @@ checksum = "b3638bc4617f96675973253b3a45006933bde93c2fd8a6170b33c777cc389e5b" dependencies = [ "async-trait", "base64 0.22.1", - "http-body", - "hyper", - "hyper-rustls", + "http-body 1.0.1", + "hyper 1.5.2", + "hyper-rustls 0.27.5", "hyper-util", "jsonrpsee-core", "jsonrpsee-types", - "rustls", + "rustls 0.23.21", "rustls-platform-verifier", "serde", "serde_json", @@ -2651,7 +3283,7 @@ version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a178c60086f24cc35bb82f57c651d0d25d99c4742b4d335de04e97fa1f08a8a1" dependencies = [ - "http", + "http 1.2.0", "serde", "serde_json", "thiserror 1.0.69", @@ -2674,7 +3306,7 @@ version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fe322e0896d0955a3ebdd5bf813571c53fea29edd713bc315b76620b327e86d" dependencies = [ - "http", + "http 1.2.0", "jsonrpsee-client-transport", "jsonrpsee-core", "jsonrpsee-types", @@ -2843,6 +3475,15 @@ version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "lzma-sys" version = "0.1.20" @@ -3361,6 +4002,12 @@ dependencies = [ "syn", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "overload" version = "0.1.1" @@ -3373,6 +4020,17 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -3632,7 +4290,7 @@ dependencies = [ "fs-err", "fs_extra", "futures", - "http", + "http 1.2.0", "human_bytes", "humantime", "ignore", @@ -3692,6 +4350,7 @@ dependencies = [ "strsim", "tabwriter", "tar", + "temp-env", "tempfile", "thiserror 2.0.11", "tokio", @@ -3822,6 +4481,7 @@ dependencies = [ "pixi_consts", "rattler", "rattler_conda_types", + "rattler_networking", "rattler_repodata_gateway", "reqwest-middleware", "rstest", @@ -4059,6 +4719,16 @@ dependencies = [ "uv-types", ] +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.31" @@ -4333,7 +5003,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.21", "socket2", "thiserror 2.0.11", "tokio", @@ -4351,7 +5021,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls", + "rustls 0.23.21", "rustls-pki-types", "slab", "thiserror 2.0.11", @@ -4604,13 +5274,15 @@ checksum = "84dd2b0aacfa246c12f98b752c53aa7f05e623290b740903dc11addc7b1ee8d9" dependencies = [ "anyhow", "async-trait", + "aws-config", + "aws-sdk-s3", "base64 0.22.1", "dirs", "fs-err", "getrandom", "google-cloud-auth", "google-cloud-token", - "http", + "http 1.2.0", "itertools 0.14.0", "keyring", "netrc-rs", @@ -4685,7 +5357,7 @@ dependencies = [ "fs-err", "futures", "hex", - "http", + "http 1.2.0", "http-cache-semantics", "humansize", "humantime", @@ -4891,6 +5563,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -4931,12 +5609,12 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.5.2", + "hyper-rustls 0.27.5", "hyper-tls", "hyper-util", "ipnet", @@ -4949,9 +5627,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.21", "rustls-native-certs 0.8.1", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -4960,7 +5638,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.1", "tokio-socks", "tokio-util", "tower 0.5.2", @@ -4982,7 +5660,7 @@ checksum = "d1ccd3b55e711f91a9885a2fa6fbbb2e39db1776420b062efc058c6410f7e5e3" dependencies = [ "anyhow", "async-trait", - "http", + "http 1.2.0", "reqwest", "serde", "thiserror 1.0.69", @@ -4999,8 +5677,8 @@ dependencies = [ "async-trait", "futures", "getrandom", - "http", - "hyper", + "http 1.2.0", + "hyper 1.5.2", "parking_lot 0.11.2", "reqwest", "reqwest-middleware", @@ -5037,6 +5715,17 @@ dependencies = [ "rand", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.8" @@ -5188,6 +5877,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.21" @@ -5198,11 +5899,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.7.3" @@ -5210,7 +5923,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework 2.11.1", @@ -5228,6 +5941,15 @@ dependencies = [ "security-framework 3.2.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -5257,10 +5979,10 @@ dependencies = [ "jni", "log", "once_cell", - "rustls", + "rustls 0.23.21", "rustls-native-certs 0.7.3", "rustls-platform-verifier-android", - "rustls-webpki", + "rustls-webpki 0.102.8", "security-framework 2.11.1", "security-framework-sys", "webpki-roots", @@ -5273,6 +5995,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -5365,12 +6097,36 @@ dependencies = [ "syn", ] +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "secret-service" version = "4.0.0" @@ -5699,6 +6455,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "simd-adler32" version = "0.3.7" @@ -5834,6 +6600,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "ssri" version = "9.2.0" @@ -6042,9 +6818,19 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc12939a1c9b9d391e0b7135f72fd30508b73450753e28341fed159317582a77" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" + +[[package]] +name = "temp-env" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" +dependencies = [ + "futures", + "parking_lot 0.12.3", +] [[package]] name = "tempfile" @@ -6251,13 +7037,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls", + "rustls 0.23.21", "tokio", ] @@ -6597,7 +7393,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "futures", - "http", + "http 1.2.0", "percent-encoding", "reqwest", "reqwest-middleware", @@ -6737,7 +7533,7 @@ dependencies = [ "fs-err", "futures", "html-escape", - "http", + "http 1.2.0", "itertools 0.14.0", "jiff", "percent-encoding", @@ -7591,6 +8387,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -8272,6 +9074,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index f130cdc7d..b156e65ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,7 @@ strsim = "0.11.1" strum = "0.26.3" tabwriter = "1.4.0" tar = "0.4.43" +temp-env = { version = "0.3.6", features = ["async_closure"] } tempfile = "3.14.0" thiserror = "2.0.9" tokio = "1.42.0" @@ -311,6 +312,7 @@ shlex = { workspace = true } strsim = { workspace = true } tabwriter = { workspace = true, features = ["ansi_formatting"] } tar = { workspace = true } +temp-env = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] } diff --git a/crates/pixi_config/Cargo.toml b/crates/pixi_config/Cargo.toml index 86ec5a0b2..81f49d5ea 100644 --- a/crates/pixi_config/Cargo.toml +++ b/crates/pixi_config/Cargo.toml @@ -19,6 +19,7 @@ miette = { workspace = true } pixi_consts = { workspace = true } rattler = { workspace = true } rattler_conda_types = { workspace = true } +rattler_networking = { workspace = true, features = ["s3"] } rattler_repodata_gateway = { workspace = true, features = ["gateway"] } reqwest-middleware = { workspace = true } serde = { workspace = true } diff --git a/crates/pixi_config/src/lib.rs b/crates/pixi_config/src/lib.rs index 45e612cdd..c8e481bee 100644 --- a/crates/pixi_config/src/lib.rs +++ b/crates/pixi_config/src/lib.rs @@ -6,6 +6,7 @@ use rattler_conda_types::{ version_spec::{EqualityOperator, LogicalOperator, RangeOperator}, ChannelConfig, NamedChannelOrUrl, Version, VersionBumpType, VersionSpec, }; +use rattler_networking::s3_middleware; use rattler_repodata_gateway::{Gateway, SourceConfig}; use reqwest_middleware::ClientWithMiddleware; use serde::{de::IntoDeserializer, Deserialize, Serialize}; @@ -277,6 +278,19 @@ pub struct PyPIConfig { pub allow_insecure_host: Vec, } +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct S3Options { + /// S3 endpoint URL + pub endpoint_url: Url, + + /// The name of the S3 region + pub region: String, + + /// Force path style URLs instead of subdomain style + pub force_path_style: bool, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] #[serde(untagged)] pub enum DetachedEnvironments { @@ -597,6 +611,11 @@ pub struct Config { #[serde(skip_serializing_if = "PyPIConfig::is_default")] pub pypi_config: PyPIConfig, + /// Configuration for S3. + #[serde(default)] + #[serde(skip_serializing_if = "HashMap::is_empty")] + pub s3_options: HashMap, + /// The option to specify the directory where detached environments are /// stored. When using 'true', it defaults to the cache directory. /// When using a path, it uses the specified path. @@ -633,6 +652,7 @@ impl Default for Config { channel_config: default_channel_config(), repodata_config: RepodataConfig::default(), pypi_config: PyPIConfig::default(), + s3_options: HashMap::new(), detached_environments: None, pinning_strategy: None, force_activate: None, @@ -923,6 +943,11 @@ impl Config { "pypi-config.index-url", "pypi-config.extra-index-urls", "pypi-config.keyring-provider", + "s3-options", + "s3-options.", + "s3-options..endpoint-url", + "s3-options..region", + "s3-options..force-path-style", "experimental.use-environment-activation-cache", ] } @@ -952,6 +977,12 @@ impl Config { channel_config: other.channel_config, repodata_config: self.repodata_config.merge(other.repodata_config), pypi_config: self.pypi_config.merge(other.pypi_config), + s3_options: { + let mut merged = HashMap::new(); + merged.extend(self.s3_options); + merged.extend(other.s3_options); + merged + }, detached_environments: other.detached_environments.or(self.detached_environments), pinning_strategy: other.pinning_strategy.or(self.pinning_strategy), force_activate: other.force_activate, @@ -1153,6 +1184,63 @@ impl Config { _ => return Err(err), } } + key if key.starts_with("s3-options") => { + if key == "s3-options" { + if let Some(value) = value { + self.s3_options = serde_json::de::from_str(&value).into_diagnostic()?; + } else { + return Err(miette!("s3-options requires a value")); + } + return Ok(()); + } + let Some(subkey) = key.strip_prefix("s3-options.") else { + return Err(err); + }; + if let Some((bucket, rest)) = subkey.split_once('.') { + if let Some(bucket_config) = self.s3_options.get_mut(bucket) { + match rest { + "endpoint-url" => { + if let Some(value) = value { + bucket_config.endpoint_url = + Url::parse(&value).into_diagnostic()?; + } else { + return Err(miette!( + "s3-options.{}.endpoint-url requires a value", + bucket + )); + } + } + "region" => { + if let Some(value) = value { + bucket_config.region = value; + } else { + return Err(miette!( + "s3-options.{}.region requires a value", + bucket + )); + } + } + "force-path-style" => { + if let Some(value) = value { + bucket_config.force_path_style = + value.parse().into_diagnostic()?; + } else { + return Err(miette!( + "s3-options.{}.force-path-style requires a value", + bucket + )); + } + } + _ => return Err(err), + } + } + } else { + let value = value.ok_or_else(|| miette!("s3-options requires a value"))?; + let s3_options: S3Options = + serde_json::de::from_str(&value).into_diagnostic()?; + self.s3_options.insert(subkey.to_string(), s3_options); + } + } key if key.starts_with(EXPERIMENTAL) => { if key == EXPERIMENTAL { if let Some(value) = value { @@ -1245,6 +1333,23 @@ impl Config { .with_max_concurrent_requests(self.max_concurrent_downloads()) .finish() } + + pub fn compute_s3_config(&self) -> HashMap { + self.s3_options + .clone() + .iter() + .map(|(k, v)| { + ( + k.clone(), + s3_middleware::S3Config::Custom { + endpoint_url: v.endpoint_url.clone(), + region: v.region.clone(), + force_path_style: v.force_path_style, + }, + ) + }) + .collect() + } } /// Returns the path to the system-level pixi config file. @@ -1394,6 +1499,41 @@ UNUSED = "unused" ); } + #[test] + fn test_s3_options_parse() { + let toml = r#" + [s3-options.bucket1] + endpoint-url = "https://my-s3-host" + region = "us-east-1" + force-path-style = false + "#; + let (config, _) = Config::from_toml(toml).unwrap(); + let s3_options = config.s3_options; + assert_eq!( + s3_options["bucket1"].endpoint_url, + Url::parse("https://my-s3-host").unwrap() + ); + assert_eq!(s3_options["bucket1"].region, "us-east-1"); + assert!(!s3_options["bucket1"].force_path_style); + } + + #[test] + fn test_s3_options_invalid_config() { + let toml = r#" + [s3-options.bucket1] + endpoint-url = "https://my-s3-host" + region = "us-east-1" + # force-path-style = false + "#; + let result = Config::from_toml(toml); + assert!(result.is_err()); + assert!(result + .err() + .unwrap() + .to_string() + .contains("missing field `force-path-style`")); + } + #[test] fn test_default_config() { let config = Config::default(); @@ -1435,6 +1575,14 @@ UNUSED = "unused" index_url: Some(Url::parse("https://conda.anaconda.org/conda-forge").unwrap()), keyring_provider: Some(KeyringProvider::Subprocess), }, + s3_options: HashMap::from([( + "bucket1".into(), + S3Options { + endpoint_url: Url::parse("https://my-s3-host").unwrap(), + region: "us-east-1".to_string(), + force_path_style: false, + }, + )]), repodata_config: RepodataConfig { default: RepodataChannelConfig { disable_bzip2: Some(true), @@ -1464,6 +1612,24 @@ UNUSED = "unused" solves: 5, ..ConcurrencyConfig::default() }, + s3_options: HashMap::from([ + ( + "bucket1".into(), + S3Options { + endpoint_url: Url::parse("https://my-s3-host").unwrap(), + region: "us-east-1".to_string(), + force_path_style: false, + }, + ), + ( + "bucket2".into(), + S3Options { + endpoint_url: Url::parse("https://my-s3-host").unwrap(), + region: "us-east-1".to_string(), + force_path_style: false, + }, + ), + ]), ..Default::default() }; config = config.merge_config(other); @@ -1476,6 +1642,7 @@ UNUSED = "unused" config.detached_environments().path().unwrap(), Some(PathBuf::from("/path/to/envs")) ); + assert!(config.s3_options.contains_key("bucket1")); let other2 = Config { default_channels: vec![NamedChannelOrUrl::from_str("channel").unwrap()], @@ -1484,6 +1651,14 @@ UNUSED = "unused" detached_environments: Some(DetachedEnvironments::Path(PathBuf::from( "/path/to/envs2", ))), + s3_options: HashMap::from([( + "bucket2".into(), + S3Options { + endpoint_url: Url::parse("https://my-new-s3-host").unwrap(), + region: "us-east-1".to_string(), + force_path_style: false, + }, + )]), ..Default::default() }; @@ -1498,6 +1673,12 @@ UNUSED = "unused" Some(PathBuf::from("/path/to/envs2")) ); assert_eq!(config.max_concurrent_solves(), 5); + assert!(config.s3_options.contains_key("bucket1")); + assert!(config.s3_options.contains_key("bucket2")); + assert!(config.s3_options["bucket2"] + .endpoint_url + .to_string() + .contains("my-new-s3-host")); let d = Path::new(&env!("CARGO_MANIFEST_DIR")) .join("tests") @@ -1513,6 +1694,7 @@ UNUSED = "unused" let mut merged = config_1.clone(); merged = merged.merge_config(config_2); + assert!(merged.s3_options.contains_key("bucket1")); let debug = format!("{:#?}", merged); let debug = debug.replace("\\\\", "/"); @@ -1691,6 +1873,15 @@ UNUSED = "unused" assert_eq!(config.max_concurrent_downloads(), 1); + config.set("s3-options.my-bucket", Some(r#"{"endpoint-url": "http://localhost:9000", "force-path-style": true, "region": "auto"}"#.to_string())).unwrap(); + let s3_options = config.s3_options.get("my-bucket").unwrap(); + assert!(s3_options + .endpoint_url + .to_string() + .contains("http://localhost:9000")); + assert!(s3_options.force_path_style); + assert_eq!(s3_options.region, "auto"); + config.set("unknown-key", None).unwrap_err(); } diff --git a/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap b/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap index e87d66b05..823d2bcad 100644 --- a/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap +++ b/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap @@ -1,7 +1,7 @@ --- source: crates/pixi_config/src/lib.rs +assertion_line: 1698 expression: debug -snapshot_kind: text --- Config { default_channels: [ @@ -97,6 +97,27 @@ Config { keyring_provider: None, allow_insecure_host: [], }, + s3_options: { + "bucket1": S3Options { + endpoint_url: Url { + scheme: "https", + cannot_be_a_base: false, + username: "", + password: None, + host: Some( + Domain( + "my-s3-host", + ), + ), + port: None, + path: "/", + query: None, + fragment: None, + }, + region: "us-east-1", + force_path_style: false, + }, + }, detached_environments: Some( Boolean( true, diff --git a/crates/pixi_config/tests/config/config_1.toml b/crates/pixi_config/tests/config/config_1.toml index d647bf9b9..853749035 100644 --- a/crates/pixi_config/tests/config/config_1.toml +++ b/crates/pixi_config/tests/config/config_1.toml @@ -4,5 +4,10 @@ tls_no_verify = true # Hardcode as we don't want this to depend on the system in the tests concurrency.solves = 1 +[s3-options.bucket1] +endpoint-url = "https://my-s3-host" +force-path-style = false +region = "us-east-1" + [mirrors] "https://conda.anaconda.org/conda-forge" = ["whatever://config_1"] diff --git a/crates/pixi_manifest/src/feature.rs b/crates/pixi_manifest/src/feature.rs index 428bd5244..6eaa51d4b 100644 --- a/crates/pixi_manifest/src/feature.rs +++ b/crates/pixi_manifest/src/feature.rs @@ -170,7 +170,6 @@ impl Feature { channel_priority: None, system_requirements: SystemRequirements::default(), pypi_options: None, - targets: as Default>::default(), } } diff --git a/crates/pixi_manifest/src/lib.rs b/crates/pixi_manifest/src/lib.rs index 38e61be44..ae9d6cbfd 100644 --- a/crates/pixi_manifest/src/lib.rs +++ b/crates/pixi_manifest/src/lib.rs @@ -15,6 +15,7 @@ mod package; mod preview; pub mod pypi; pub mod pyproject; +mod s3; mod solve_group; mod spec_type; mod system_requirements; @@ -48,6 +49,7 @@ use miette::Diagnostic; pub use preview::{KnownPreviewFeature, Preview}; pub use pypi::pypi_requirement::PyPiRequirement; use rattler_conda_types::Platform; +pub use s3::S3Options; pub use spec_type::SpecType; pub use system_requirements::{LibCFamilyAndVersion, LibCSystemRequirement, SystemRequirements}; pub use target::{PackageTarget, TargetSelector, Targets, WorkspaceTarget}; diff --git a/crates/pixi_manifest/src/s3.rs b/crates/pixi_manifest/src/s3.rs new file mode 100644 index 000000000..beacf729a --- /dev/null +++ b/crates/pixi_manifest/src/s3.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; +use url::Url; + +/// Custom S3 configuration +#[derive(Debug, Clone, PartialEq, Serialize, Eq, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct S3Options { + /// S3 endpoint URL + pub endpoint_url: Url, + /// Name of the region + pub region: String, + /// Force path style URLs instead of subdomain style + pub force_path_style: bool, +} diff --git a/crates/pixi_manifest/src/toml/mod.rs b/crates/pixi_manifest/src/toml/mod.rs index 8857897ff..5fba73c7d 100644 --- a/crates/pixi_manifest/src/toml/mod.rs +++ b/crates/pixi_manifest/src/toml/mod.rs @@ -10,6 +10,7 @@ mod platform; mod preview; mod pypi_options; pub mod pyproject; +mod s3_options; mod system_requirements; mod target; mod task; diff --git a/crates/pixi_manifest/src/toml/s3_options.rs b/crates/pixi_manifest/src/toml/s3_options.rs new file mode 100644 index 000000000..ee45257bd --- /dev/null +++ b/crates/pixi_manifest/src/toml/s3_options.rs @@ -0,0 +1,23 @@ +use crate::S3Options; + +use pixi_toml::TomlFromStr; +use toml_span::{de_helpers::TableHelper, DeserError, Value}; + +impl<'de> toml_span::Deserialize<'de> for S3Options { + fn deserialize(value: &mut Value<'de>) -> Result { + let mut th = TableHelper::new(value)?; + + let endpoint_url = th + .required::>("endpoint-url") + .map(TomlFromStr::into_inner)?; + let region = th.required("region")?; + let force_path_style = th.required("force-path-style")?; + th.finalize(None)?; + + Ok(Self { + endpoint_url, + region, + force_path_style, + }) + } +} diff --git a/crates/pixi_manifest/src/toml/workspace.rs b/crates/pixi_manifest/src/toml/workspace.rs index 577e26909..b6b16d32a 100644 --- a/crates/pixi_manifest/src/toml/workspace.rs +++ b/crates/pixi_manifest/src/toml/workspace.rs @@ -16,7 +16,7 @@ use crate::{ toml::{platform::TomlPlatform, preview::TomlPreview}, utils::PixiSpanned, workspace::ChannelPriority, - PrioritizedChannel, TargetSelector, Targets, TomlError, WithWarnings, Workspace, + PrioritizedChannel, S3Options, TargetSelector, Targets, TomlError, WithWarnings, Workspace, }; #[derive(Debug, Clone)] @@ -45,6 +45,7 @@ pub struct TomlWorkspace { pub documentation: Option, pub conda_pypi_map: Option>, pub pypi_options: Option, + pub s3_options: Option>, pub preview: TomlPreview, pub target: IndexMap, TomlWorkspaceTarget>, pub build_variants: Option>>, @@ -129,6 +130,7 @@ impl TomlWorkspace { platforms: self.platforms.value, conda_pypi_map: self.conda_pypi_map, pypi_options: self.pypi_options, + s3_options: self.s3_options, preview, build_variants: Targets::from_default_and_user_defined( self.build_variants, @@ -180,6 +182,9 @@ impl<'de> toml_span::Deserialize<'de> for TomlWorkspace { .optional::>("conda-pypi-map") .map(TomlHashMap::into_inner); let pypi_options = th.optional("pypi-options"); + let s3_options = th + .optional::>("s3-options") + .map(TomlHashMap::into_inner); let preview = th.optional("preview").unwrap_or_default(); let target = th .optional::>("target") @@ -206,6 +211,7 @@ impl<'de> toml_span::Deserialize<'de> for TomlWorkspace { documentation, conda_pypi_map, pypi_options, + s3_options, preview, target: target.unwrap_or_default(), build_variants, diff --git a/crates/pixi_manifest/src/workspace.rs b/crates/pixi_manifest/src/workspace.rs index 93d89ebae..c1bbec974 100644 --- a/crates/pixi_manifest/src/workspace.rs +++ b/crates/pixi_manifest/src/workspace.rs @@ -8,7 +8,7 @@ use toml_span::{DeserError, Value}; use url::Url; use super::pypi::pypi_options::PypiOptions; -use crate::{preview::Preview, PrioritizedChannel, Targets}; +use crate::{preview::Preview, PrioritizedChannel, S3Options, Targets}; /// Describes the contents of the `[workspace]` section of the project manifest. #[derive(Debug, Clone)] @@ -58,6 +58,9 @@ pub struct Workspace { /// The pypi options supported in the project pub pypi_options: Option, + /// The S3 options supported in the project + pub s3_options: Option>, + /// Preview features pub preview: Preview, diff --git a/crates/pixi_utils/Cargo.toml b/crates/pixi_utils/Cargo.toml index 432451685..2a5895805 100644 --- a/crates/pixi_utils/Cargo.toml +++ b/crates/pixi_utils/Cargo.toml @@ -33,7 +33,7 @@ pep508_rs = { workspace = true } pixi_config = { workspace = true } pixi_consts = { workspace = true } rattler_conda_types = { workspace = true } -rattler_networking = { workspace = true, features = ["gcs"] } +rattler_networking = { workspace = true, features = ["gcs", "s3"] } reqwest = { workspace = true } reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } diff --git a/crates/pixi_utils/src/reqwest.rs b/crates/pixi_utils/src/reqwest.rs index 9cc68d65f..3b7fa734f 100644 --- a/crates/pixi_utils/src/reqwest.rs +++ b/crates/pixi_utils/src/reqwest.rs @@ -1,18 +1,20 @@ use std::{any::Any, path::PathBuf, sync::Arc, time::Duration}; +use miette::IntoDiagnostic; use pixi_consts::consts; use rattler_networking::{ authentication_storage::{self, AuthenticationStorageError}, mirror_middleware::Mirror, retry_policies::ExponentialBackoff, AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, MirrorMiddleware, - OciMiddleware, + OciMiddleware, S3Middleware, }; use reqwest::Client; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_retry::RetryTransientMiddleware; use std::collections::HashMap; +use tracing::debug; use pixi_config::Config; @@ -97,7 +99,10 @@ pub fn oci_middleware() -> OciMiddleware { OciMiddleware } -pub fn build_reqwest_clients(config: Option<&Config>) -> (Client, ClientWithMiddleware) { +pub fn build_reqwest_clients( + config: Option<&Config>, + s3_config_project: Option>, +) -> miette::Result<(Client, ClientWithMiddleware)> { let app_user_agent = format!("pixi/{}", consts::PIXI_VERSION); // If we do not have a config, we will just load the global default. @@ -131,6 +136,18 @@ pub fn build_reqwest_clients(config: Option<&Config>) -> (Client, ClientWithMidd client_builder = client_builder.with(GCSMiddleware); + let s3_config_global = config.compute_s3_config(); + let s3_config_project = s3_config_project.unwrap_or_default(); + let mut s3_config = HashMap::new(); + s3_config.extend(s3_config_global); + s3_config.extend(s3_config_project); + + debug!("Using s3_config: {:?}", s3_config); + let store = auth_store(&config).into_diagnostic()?; + let s3_middleware = S3Middleware::new(s3_config, store); + debug!("s3_middleware: {:?}", s3_middleware); + client_builder = client_builder.with(s3_middleware); + client_builder = client_builder.with_arc(Arc::new( auth_middleware(&config).expect("could not create auth middleware"), )); @@ -141,5 +158,5 @@ pub fn build_reqwest_clients(config: Option<&Config>) -> (Client, ClientWithMidd let authenticated_client = client_builder.build(); - (client, authenticated_client) + Ok((client, authenticated_client)) } diff --git a/docs/advanced/authentication.md b/docs/advanced/authentication.md index 7c04bab4b..8da81fcd3 100644 --- a/docs/advanced/authentication.md +++ b/docs/advanced/authentication.md @@ -15,13 +15,18 @@ Arguments: The host to authenticate with (e.g. repo.prefix.dev) Options: - --token The token to use (for authentication with prefix.dev) - --username The username to use (for basic HTTP authentication) - --password The password to use (for basic HTTP authentication) - --conda-token The token to use on anaconda.org / quetz authentication - -v, --verbose... More output per occurrence - -q, --quiet... Less output per occurrence - -h, --help Print help + --token The token to use (for authentication with prefix.dev) + --username The username to use (for basic HTTP authentication) + --password The password to use (for basic HTTP authentication) + --conda-token The token to use on anaconda.org / quetz authentication + --s3-access-key-id The S3 access key ID + --s3-secret-access-key The S3 secret access key + --s3-session-token The S3 session token + -v, --verbose... Increase logging verbosity + -q, --quiet... Decrease logging verbosity + --color Whether the log needs to be colored [env: PIXI_COLOR=] [default: auto] [possible values: always, never, auto] + --no-progress Hide all progress bars, always turned on if stderr is not a terminal [env: PIXI_NO_PROGRESS=] + -h, --help Print help ``` The different options are "token", "conda-token" and "username + password". @@ -53,6 +58,12 @@ Login to a basic HTTP secured server: pixi auth login myserver.com --username user --password password ``` +Login to a S3 bucket: + +```shell +pixi auth login s3://my-bucket --s3-access-key-id accesskey --s3-secret-access-key secretkey +``` + ## Where does pixi store the authentication information? The storage location for the authentication information is system-dependent. By default, pixi tries to use the keychain to store this sensitive information securely on your machine. @@ -108,6 +119,7 @@ Note: if you use a wildcard in the host, any subdomain will match (e.g. `*.prefi Lastly you can set the authentication override file in the [global configuration file](./../reference/pixi_configuration.md). ## PyPI authentication + Currently, we support the following methods for authenticating against PyPI: 1. [keyring](https://pypi.org/project/keyring/) authentication. @@ -120,6 +132,7 @@ We want to add more methods in the future, so if you have a specific method you Currently, pixi supports the uv method of authentication through the python [keyring](https://pypi.org/project/keyring/) library. #### Installing keyring + To install keyring you can use pixi global install: === "Basic Auth" @@ -190,6 +203,7 @@ For other registries, you will need to adapt these instructions to add the right ``` #### Installing your environment + Either configure your [Global Config](../reference/pixi_configuration.md#pypi-config), or use the flag `--pypi-keyring-provider` which can either be set to `subprocess` (activated) or `disabled`: ```shell @@ -197,7 +211,6 @@ Either configure your [Global Config](../reference/pixi_configuration.md#pypi-co pixi install --pypi-keyring-provider subprocess ``` - ### `.netrc` file `pixi` allows you to access private registries securely by authenticating with credentials stored in a `.netrc` file. @@ -214,4 +227,5 @@ machine registry-name login admin password admin ``` + For more details, you can access the [.netrc docs](https://www.ibm.com/docs/en/aix/7.2?topic=formats-netrc-file-format-tcpip). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3f0434ac3..6b2734e50 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -919,11 +919,15 @@ Store authentication information for given host. - `--username `: The username to use for basic HTTP authentication - `--password `: The password to use for basic HTTP authentication. - `--conda-token `: The token to use on `anaconda.org` / `quetz` authentication. +- `--s3-access-key-id`: The S3 access key ID +- `--s3-secret-access-key`: The S3 secret access key +- `--s3-session-token`: The S3 session token (optional for S3 authentication) ```shell pixi auth login repo.prefix.dev --token pfx_JQEV-m_2bdz-D8NSyRSaAndHANx0qHjq7f2iD pixi auth login anaconda.org --conda-token ABCDEFGHIJKLMNOP pixi auth login https://myquetz.server --username john --password xxxxxx +pixi auth login s3://my-bucket --s3-access-key-id $AWS_ACCESS_KEY_ID --s3-access-key-id $AWS_SECRET_KEY_ID ``` ### `auth logout` @@ -938,6 +942,7 @@ Remove authentication information for a given host. pixi auth logout pixi auth logout repo.prefix.dev pixi auth logout anaconda.org +pixi auth logout s3://my-bucket ``` ## `config` @@ -1031,6 +1036,7 @@ pixi config set --global mirrors '{"https://conda.anaconda.org/": ["https://pref pixi config set repodata-config.disable-zstd true --system pixi config set --global detached-environments "/opt/pixi/envs" pixi config set detached-environments false +pixi config set s3-options.my-bucket '{"endpoint-url": "http://localhost:9000", "force-path-style": true, "region": "auto"}' ``` ### `config unset` diff --git a/pixi.toml b/pixi.toml index a994812db..b133e63cc 100644 --- a/pixi.toml +++ b/pixi.toml @@ -110,7 +110,7 @@ compilers = ">=1.9.0,<2" git = ">=2.46.0,<3" openssl = "3.*" pkg-config = "0.29.*" -rust = ">=1.84.0,<1.85" +rust = "==1.84.0" [feature.build.target.linux-64.dependencies] clang = ">=18.1.8,<19.0" diff --git a/schema/model.py b/schema/model.py index 5ad0dc07d..d0fb05663 100644 --- a/schema/model.py +++ b/schema/model.py @@ -154,6 +154,9 @@ class Workspace(StrictBaseModel): pypi_options: PyPIOptions | None = Field( None, description="Options related to PyPI indexes for this project" ) + s3_options: dict[str, S3Options] | None = Field( + None, description="Options related to S3 for this project" + ) preview: list[KnownPreviewFeature | str] | bool | None = Field( None, description="Defines the enabling of preview features of the project" ) @@ -507,6 +510,22 @@ class FindLinksURL(StrictBaseModel): ) +class S3Options(StrictBaseModel): + """Options related to S3 for this project""" + + endpoint_url: NonEmptyStr = Field( + description="The endpoint URL to use for the S3 client", + examples=["https://s3.eu-central-1.amazonaws.com"], + ) + region: NonEmptyStr = Field( + description="The region to use for the S3 client", + examples=["eu-central-1"], + ) + force_path_style: bool = Field( + description="Whether to force path style for the S3 client", + ) + + class PyPIOptions(StrictBaseModel): """Options that determine the behavior of PyPI package resolution and installation""" diff --git a/schema/schema.json b/schema/schema.json index f17a52a6d..b809d3f8d 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -1292,6 +1292,42 @@ } } }, + "S3Options": { + "title": "S3Options", + "description": "Options related to S3 for this project", + "type": "object", + "required": [ + "endpoint-url", + "region", + "force-path-style" + ], + "additionalProperties": false, + "properties": { + "endpoint-url": { + "title": "Endpoint-Url", + "description": "The endpoint URL to use for the S3 client", + "type": "string", + "minLength": 1, + "examples": [ + "https://s3.eu-central-1.amazonaws.com" + ] + }, + "force-path-style": { + "title": "Force-Path-Style", + "description": "Whether to force path style for the S3 client", + "type": "boolean" + }, + "region": { + "title": "Region", + "description": "The region to use for the S3 client", + "type": "string", + "minLength": 1, + "examples": [ + "eu-central-1" + ] + } + } + }, "SystemRequirements": { "title": "SystemRequirements", "description": "Platform-specific requirements", @@ -1776,6 +1812,14 @@ "format": "uri", "minLength": 1 }, + "s3-options": { + "title": "S3-Options", + "description": "Options related to S3 for this project", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/S3Options" + } + }, "version": { "title": "Version", "description": "The version of the project; we advise use of [SemVer](https://semver.org)", diff --git a/src/cli/build.rs b/src/cli/build.rs index dd5352900..703a86c0b 100644 --- a/src/cli/build.rs +++ b/src/cli/build.rs @@ -89,8 +89,8 @@ pub async fn execute(args: Args) -> miette::Result<()> { let channel_config = workspace.channel_config(); let tool_context = pixi_build_frontend::ToolContext::builder() - .with_gateway(workspace.repodata_gateway().clone()) - .with_client(workspace.authenticated_client().clone()) + .with_gateway(workspace.repodata_gateway()?.clone()) + .with_client(workspace.authenticated_client()?.clone()) .build(); let protocol = pixi_build_frontend::BuildFrontend::default() diff --git a/src/cli/exec.rs b/src/cli/exec.rs index b030d6727..89b87f6b9 100644 --- a/src/cli/exec.rs +++ b/src/cli/exec.rs @@ -54,7 +54,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { let mut command_args = args.command.iter(); let command = command_args.next().ok_or_else(|| miette::miette!(help ="i.e when specifying specs explicitly use a command at the end: `pixi exec -s python==3.12 python`", "missing required command to execute",))?; - let (_, client) = build_reqwest_clients(Some(&config)); + let (_, client) = build_reqwest_clients(Some(&config), None)?; // Create the environment to run the command in. let prefix = create_exec_prefix(&args, &cache_dir, &config, &client).await?; diff --git a/src/cli/init.rs b/src/cli/init.rs index d65ded9af..11eba2d64 100644 --- a/src/cli/init.rs +++ b/src/cli/init.rs @@ -82,6 +82,25 @@ version = "{{ version }}" {% if extra_index_urls %}extra-index-urls = {{ extra_index_urls }}{% endif %} {%- endif %} +{%- if s3 %} +{%- for key in s3 %} + +[project.s3-options.{{ key }}] +{%- if s3[key]["endpoint-url"] %} +endpoint-url = "{{ s3[key]["endpoint-url"] }}" +{%- endif %} +{%- if s3[key].region %} +{%- endif %} +{%- if s3[key].region %} +region = "{{ s3[key].region }}" +{%- endif %} +{%- if s3[key]["force-path-style"] is not none %} +force-path-style = {{ s3[key]["force-path-style"] }} +{%- endif %} + +{%- endfor %} +{%- endif %} + [tasks] [dependencies] @@ -116,6 +135,25 @@ default = { solve-group = "default" } {{env}} = { features = {{ features }}, solve-group = "default" } {%- endfor %} +{%- if s3 %} +{%- for key in s3 %} + +[tool.pixi.project.s3-options.{{ key }}] +{%- if s3[key]["endpoint-url"] %} +endpoint-url = "{{ s3[key]["endpoint-url"] }}" +{%- endif %} +{%- if s3[key].region %} +{%- endif %} +{%- if s3[key].region %} +region = "{{ s3[key].region }}" +{%- endif %} +{%- if s3[key]["force-path-style"] is not none %} +force-path-style = {{ s3[key]["force-path-style"] }} +{%- endif %} + +{%- endfor %} +{%- endif %} + [tool.pixi.tasks] "#; @@ -148,6 +186,25 @@ platforms = {{ platforms }} {% if extra_index_urls %}extra-index-urls = {{ extra_index_urls }}{% endif %} {%- endif %} +{%- if s3 %} +{%- for key in s3 %} + +[tool.pixi.project.s3-options.{{ key }}] +{%- if s3[key]["endpoint-url"] %} +endpoint-url = "{{ s3[key]["endpoint-url"] }}" +{%- endif %} +{%- if s3[key].region %} +{%- endif %} +{%- if s3[key].region %} +region = "{{ s3[key].region }}" +{%- endif %} +{%- if s3[key]["force-path-style"] is not none %} +force-path-style = {{ s3[key]["force-path-style"] }} +{%- endif %} + +{%- endfor %} +{%- endif %} + [tool.pixi.pypi-dependencies] {{ pypi_package_name }} = { path = ".", editable = true } @@ -243,6 +300,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { &platforms, None, &vec![], + config.s3_options, Some(&env_vars), ); let mut workspace = @@ -343,6 +401,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { channels, platforms, environments, + s3 => relevant_s3_options(config.s3_options, channels), }, ) .expect("should be able to render the template"); @@ -398,6 +457,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { platforms, index_url => index_url.as_ref(), extra_index_urls => &extra_index_urls, + s3 => relevant_s3_options(config.s3_options, channels), }, ) .expect("should be able to render the template"); @@ -443,6 +503,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { &platforms, index_url.as_ref(), &extra_index_urls, + config.s3_options, None, ); save_manifest_file(&pixi_manifest_path, rv)?; @@ -482,6 +543,7 @@ fn render_project( platforms: &Vec, index_url: Option<&Url>, extra_index_urls: &Vec, + s3_options: HashMap, env_vars: Option<&HashMap>, ) -> String { let ctx = context! { @@ -492,6 +554,7 @@ fn render_project( platforms, index_url, extra_index_urls, + s3 => relevant_s3_options(s3_options, channels), env_vars => {if let Some(env_vars) = env_vars { env_vars.iter().map(|(k, v)| format!("{} = \"{}\"", k, v)).collect::>().join(", ") } else {String::new()}}, @@ -501,6 +564,32 @@ fn render_project( .expect("should be able to render the template") } +fn relevant_s3_options( + s3_options: HashMap, + channels: Vec, +) -> HashMap { + // only take s3 options in manifest if they are used in the default channels + let s3_buckets = channels + .iter() + .filter_map(|channel| match channel { + NamedChannelOrUrl::Name(_) => None, + NamedChannelOrUrl::Path(_) => None, + NamedChannelOrUrl::Url(url) => { + if url.scheme() == "s3" { + url.host().map(|host| host.to_string()) + } else { + None + } + } + }) + .collect::>(); + + s3_options + .into_iter() + .filter(|(key, _)| s3_buckets.contains(key)) + .collect() +} + /// Save the rendered template to a file, and print a message to the user. fn save_manifest_file(path: &Path, content: String) -> miette::Result<()> { fs_err::write(path, content).into_diagnostic()?; diff --git a/src/cli/search.rs b/src/cli/search.rs index d43f23375..6f9bb35f8 100644 --- a/src/cli/search.rs +++ b/src/cli/search.rs @@ -138,10 +138,12 @@ pub async fn execute_impl( let package_name_filter = args.package; - let client = project - .as_ref() - .map(|p| p.authenticated_client().clone()) - .unwrap_or_else(|| build_reqwest_clients(None).1); + let project = project.as_ref(); + let client = if let Some(project) = project { + project.authenticated_client()?.clone() + } else { + build_reqwest_clients(None, None)?.1 + }; let config = Config::load_global(); diff --git a/src/conda_prefix_updater.rs b/src/conda_prefix_updater.rs index fc1577d29..08c7ff27b 100644 --- a/src/conda_prefix_updater.rs +++ b/src/conda_prefix_updater.rs @@ -84,7 +84,7 @@ impl<'a> CondaPrefixUpdater<'a> { let has_existing_packages = !installed_packages.is_empty(); let group_name = self.group.name().clone(); - let client = self.group.workspace().authenticated_client().clone(); + let client = self.group.workspace().authenticated_client()?.clone(); let prefix = self.group.prefix(); let python_status = environment::update_prefix_conda( diff --git a/src/environment/conda_prefix.rs b/src/environment/conda_prefix.rs index 036818a41..97ea64e3c 100644 --- a/src/environment/conda_prefix.rs +++ b/src/environment/conda_prefix.rs @@ -121,7 +121,7 @@ impl<'a> CondaPrefixUpdaterBuilder<'a> { let name = self.group.name(); let prefix = self.group.prefix(); let virtual_packages = self.group.virtual_packages(self.platform); - let client = self.group.workspace().authenticated_client().clone(); + let client = self.group.workspace().authenticated_client()?.clone(); Ok(CondaPrefixUpdater::new( channels, diff --git a/src/global/project/mod.rs b/src/global/project/mod.rs index 4930c1fb6..1ad1aefa9 100644 --- a/src/global/project/mod.rs +++ b/src/global/project/mod.rs @@ -31,6 +31,7 @@ use is_executable::IsExecutable; use itertools::Itertools; pub(crate) use manifest::{ExposedType, Manifest, Mapping}; use miette::{miette, Context, IntoDiagnostic}; +use once_cell::sync::OnceCell; use parsed_manifest::ParsedManifest; pub(crate) use parsed_manifest::{ExposedName, ParsedEnvironment}; use pixi_config::{default_channel_config, pixi_home, Config}; @@ -56,7 +57,6 @@ use std::{ fmt::{Debug, Formatter}, path::{Path, PathBuf}, str::FromStr, - sync::OnceLock, }; use toml_edit::DocumentMut; use uv_configuration::RAYON_INITIALIZE; @@ -84,11 +84,13 @@ pub struct Project { /// Binary directory pub(crate) bin_dir: BinDir, /// Reqwest client shared for this project. - /// This is wrapped in a `OnceLock` to allow for lazy initialization. - client: OnceLock<(reqwest::Client, ClientWithMiddleware)>, + /// This is wrapped in a `OnceCell` to allow for lazy initialization. + // TODO: once https://github.com/rust-lang/rust/issues/109737 is stabilized, switch to OnceLock + client: OnceCell<(reqwest::Client, ClientWithMiddleware)>, /// The repodata gateway to use for answering queries about repodata. - /// This is wrapped in a `OnceLock` to allow for lazy initialization. - repodata_gateway: OnceLock, + /// This is wrapped in a `OnceCell` to allow for lazy initialization. + // TODO: once https://github.com/rust-lang/rust/issues/109737 is stabilized, switch to OnceLock + repodata_gateway: OnceCell, } impl Debug for Project { @@ -256,8 +258,8 @@ impl Project { let config = Config::load(&root); - let client = OnceLock::new(); - let repodata_gateway = OnceLock::new(); + let client = OnceCell::new(); + let repodata_gateway = OnceCell::new(); Self { root, manifest, @@ -457,13 +459,15 @@ impl Project { /// Create an authenticated reqwest client for this project /// use authentication from `rattler_networking` - pub fn authenticated_client(&self) -> &ClientWithMiddleware { - &self.client_and_authenticated_client().1 + pub fn authenticated_client(&self) -> miette::Result<&ClientWithMiddleware> { + Ok(&self.client_and_authenticated_client()?.1) } - fn client_and_authenticated_client(&self) -> &(reqwest::Client, ClientWithMiddleware) { + fn client_and_authenticated_client( + &self, + ) -> miette::Result<&(reqwest::Client, ClientWithMiddleware)> { self.client - .get_or_init(|| build_reqwest_clients(Some(&self.config))) + .get_or_try_init(|| build_reqwest_clients(Some(&self.config), None)) } pub(crate) fn config(&self) -> &Config { @@ -516,7 +520,7 @@ impl Project { env_name.fancy_display() ), |_| async { - self.repodata_gateway() + self.repodata_gateway()? .query(channels, [platform, Platform::NoArch], match_specs.clone()) .recursive(true) .await @@ -572,6 +576,7 @@ impl Project { // Install the environment let package_cache = PackageCache::new(pixi_config::get_cache_dir()?.join("pkgs")); let prefix = self.environment_prefix(env_name).await?; + let authenticated_client = self.authenticated_client()?.clone(); let result = await_in_progress( format!( "Creating virtual environment for {}", @@ -579,7 +584,7 @@ impl Project { ), |pb| { Installer::new() - .with_download_client(self.authenticated_client().clone()) + .with_download_client(authenticated_client) .with_execute_link_scripts(false) .with_package_cache(package_cache) .with_target_platform(platform) @@ -1054,9 +1059,11 @@ impl Project { impl Repodata for Project { /// Returns the [`Gateway`] used by this project. - fn repodata_gateway(&self) -> &Gateway { - self.repodata_gateway - .get_or_init(|| self.config().gateway(self.authenticated_client().clone())) + fn repodata_gateway(&self) -> miette::Result<&Gateway> { + self.repodata_gateway.get_or_try_init(|| { + let client = self.authenticated_client()?.clone(); + Ok(self.config().gateway(client)) + }) } } diff --git a/src/lock_file/resolve/uv_resolution_context.rs b/src/lock_file/resolve/uv_resolution_context.rs index e367e2c6c..eb88a8718 100644 --- a/src/lock_file/resolve/uv_resolution_context.rs +++ b/src/lock_file/resolve/uv_resolution_context.rs @@ -66,7 +66,7 @@ impl UvResolutionContext { cache, in_flight: InFlight::default(), hash_strategy: HashStrategy::None, - client: project.client().clone(), + client: project.client()?.clone(), keyring_provider, concurrency: Concurrency::default(), source_strategy: SourceStrategy::Disabled, diff --git a/src/lock_file/update.rs b/src/lock_file/update.rs index 233db9e47..5a1e80348 100644 --- a/src/lock_file/update.rs +++ b/src/lock_file/update.rs @@ -1006,8 +1006,8 @@ impl<'p> UpdateContextBuilder<'p> { }) .collect(); - let gateway = project.repodata_gateway().clone(); - let client = project.authenticated_client().clone(); + let gateway = project.repodata_gateway()?.clone(); + let client = project.authenticated_client()?.clone(); // tool context let tool_context = ToolContext::builder() @@ -1129,10 +1129,10 @@ impl<'p> UpdateContext<'p> { let group_solve_task = spawn_solve_conda_environment_task( source.clone(), locked_group_records, - project.repodata_gateway().clone(), + project.repodata_gateway()?.clone(), platform, self.conda_solve_semaphore.clone(), - project.authenticated_client().clone(), + project.authenticated_client()?.clone(), channel_priority, self.build_context.clone(), ) @@ -2039,7 +2039,7 @@ async fn spawn_solve_pypi_task<'p>( let locked_pypi_records = locked_pypi_packages.records.clone(); pypi_mapping::amend_pypi_purls( - environment.workspace().client().clone().into(), + environment.workspace().client()?.clone().into(), pypi_name_mapping_location, pixi_solve_records .iter_mut() diff --git a/src/repodata.rs b/src/repodata.rs index 1ca0e1ac8..13ca54cab 100644 --- a/src/repodata.rs +++ b/src/repodata.rs @@ -2,5 +2,5 @@ use rattler_repodata_gateway::Gateway; pub(crate) trait Repodata { /// Returns the [`Gateway`] used by this project. - fn repodata_gateway(&self) -> &Gateway; + fn repodata_gateway(&self) -> miette::Result<&Gateway>; } diff --git a/src/workspace/mod.rs b/src/workspace/mod.rs index 3564bb356..2644b45bd 100644 --- a/src/workspace/mod.rs +++ b/src/workspace/mod.rs @@ -15,7 +15,7 @@ use std::{ fmt::{Debug, Formatter}, hash::Hash, path::{Path, PathBuf}, - sync::{Arc, OnceLock}, + sync::Arc, }; use async_once_cell::OnceCell as AsyncCell; @@ -39,6 +39,7 @@ use pixi_utils::reqwest::build_reqwest_clients; use pypi_mapping::{ChannelName, CustomMapping, MappingLocation, MappingSource}; use rattler_conda_types::{Channel, ChannelConfig, MatchSpec, PackageName, Platform}; use rattler_lock::{LockFile, LockedPackageRef}; +use rattler_networking::s3_middleware; use rattler_repodata_gateway::Gateway; use reqwest_middleware::ClientWithMiddleware; pub use solve_group::SolveGroup; @@ -127,11 +128,13 @@ pub struct Workspace { /// Reqwest client shared for this workspace. /// This is wrapped in a `OnceLock` to allow for lazy initialization. - client: OnceLock<(reqwest::Client, ClientWithMiddleware)>, + // TODO: once https://github.com/rust-lang/rust/issues/109737 is stabilized, switch to OnceLock + client: OnceCell<(reqwest::Client, ClientWithMiddleware)>, /// The repodata gateway to use for answering queries about repodata. /// This is wrapped in a `OnceLock` to allow for lazy initialization. - repodata_gateway: OnceLock, + // TODO: once https://github.com/rust-lang/rust/issues/109737 is stabilized, switch to OnceLock + repodata_gateway: OnceCell, /// The manifest for the workspace pub workspace: WithProvenance, @@ -150,6 +153,8 @@ pub struct Workspace { /// The global configuration as loaded from the config file(s) config: Config, + /// The S3 configuration + s3_config: HashMap, } impl Debug for Workspace { @@ -183,6 +188,22 @@ impl Workspace { .expect("manifest path should always have a parent") .to_owned(); + let s3_options = manifest.workspace.value.workspace.s3_options.clone(); + let s3_config = s3_options + .unwrap_or_default() + .iter() + .map(|(key, value)| { + ( + key.clone(), + s3_middleware::S3Config::Custom { + endpoint_url: value.endpoint_url.clone(), + region: value.region.clone(), + force_path_style: value.force_path_style, + }, + ) + }) + .collect::>(); + let config = Config::load(&root); Self { root, @@ -192,6 +213,7 @@ impl Workspace { env_vars, mapping_source: Default::default(), config, + s3_config, repodata_gateway: Default::default(), } } @@ -404,19 +426,22 @@ impl Workspace { } /// Returns the reqwest client used for http networking - pub(crate) fn client(&self) -> &reqwest::Client { - &self.client_and_authenticated_client().0 + pub(crate) fn client(&self) -> miette::Result<&reqwest::Client> { + Ok(&self.client_and_authenticated_client()?.0) } /// Create an authenticated reqwest client for this project /// use authentication from `rattler_networking` - pub fn authenticated_client(&self) -> &ClientWithMiddleware { - &self.client_and_authenticated_client().1 + pub fn authenticated_client(&self) -> miette::Result<&ClientWithMiddleware> { + Ok(&self.client_and_authenticated_client()?.1) } - fn client_and_authenticated_client(&self) -> &(reqwest::Client, ClientWithMiddleware) { - self.client - .get_or_init(|| build_reqwest_clients(Some(&self.config))) + fn client_and_authenticated_client( + &self, + ) -> miette::Result<&(reqwest::Client, ClientWithMiddleware)> { + self.client.get_or_try_init(|| { + build_reqwest_clients(Some(&self.config), Some(self.s3_config.clone())) + }) } pub(crate) fn config(&self) -> &Config { diff --git a/src/workspace/repodata.rs b/src/workspace/repodata.rs index 9ee9975e8..cc0b5b86b 100644 --- a/src/workspace/repodata.rs +++ b/src/workspace/repodata.rs @@ -4,8 +4,8 @@ use rattler_repodata_gateway::Gateway; impl Repodata for Workspace { /// Returns the [`Gateway`] used by this project. - fn repodata_gateway(&self) -> &Gateway { + fn repodata_gateway(&self) -> miette::Result<&Gateway> { self.repodata_gateway - .get_or_init(|| self.config().gateway(self.authenticated_client().clone())) + .get_or_try_init(|| Ok(self.config().gateway(self.authenticated_client()?.clone()))) } } diff --git a/tests/integration_rust/install_tests.rs b/tests/integration_rust/install_tests.rs index c1fe3c840..5ff22417d 100644 --- a/tests/integration_rust/install_tests.rs +++ b/tests/integration_rust/install_tests.rs @@ -33,7 +33,7 @@ use std::{ str::FromStr, sync::Arc, }; -use tempfile::TempDir; +use tempfile::{tempdir, TempDir}; use tokio::{fs, task::JoinSet}; use url::Url; use uv_python::PythonEnvironment; @@ -929,7 +929,7 @@ async fn test_multiple_prefix_update() { .channel_urls(&group.workspace().channel_config()) .unwrap(); let name = group.name(); - let client = group.workspace().authenticated_client().clone(); + let client = group.workspace().authenticated_client().unwrap().clone(); let prefix = group.prefix(); let virtual_packages = group.virtual_packages(current_platform); @@ -991,3 +991,81 @@ async fn test_multiple_prefix_update() { assert_eq!(*first_modified_date, prefix_metadata.modified().unwrap()); } } + +/// Should download a package from an S3 bucket and install it +#[tokio::test] +#[cfg_attr(not(feature = "slow_integration_tests"), ignore)] +async fn install_s3() { + let r2_access_key_id = std::env::var("PIXI_TEST_R2_ACCESS_KEY_ID").ok(); + let r2_secret_access_key = std::env::var("PIXI_TEST_R2_SECRET_ACCESS_KEY").ok(); + if r2_access_key_id.is_none() + || r2_access_key_id.clone().unwrap().is_empty() + || r2_secret_access_key.is_none() + || r2_secret_access_key.clone().unwrap().is_empty() + { + eprintln!( + "Skipping test as PIXI_TEST_R2_ACCESS_KEY_ID or PIXI_TEST_R2_SECRET_ACCESS_KEY is not set" + ); + return; + } + + let r2_access_key_id = r2_access_key_id.unwrap(); + let r2_secret_access_key = r2_secret_access_key.unwrap(); + + let credentials = format!( + r#" + {{ + "s3://rattler-s3-testing/channel": {{ + "S3Credentials": {{ + "access_key_id": "{}", + "secret_access_key": "{}" + }} + }} + }} + "#, + r2_access_key_id, r2_secret_access_key + ); + let temp_dir = tempdir().unwrap(); + let credentials_path = temp_dir.path().join("credentials.json"); + let mut file = File::create(credentials_path.clone()).unwrap(); + file.write_all(credentials.as_bytes()).unwrap(); + + let manifest = format!( + r#" + [project] + name = "s3-test" + channels = ["s3://rattler-s3-testing/channel", "conda-forge"] + platforms = ["{platform}"] + + [project.s3.rattler-s3-testing] + endpoint-url = "https://e1a7cde76f1780ec06bac859036dbaf7.eu.r2.cloudflarestorage.com" + region = "auto" + force-path-style = true + + [dependencies] + my-webserver = {{ version = "0.1.0", build = "pyh4616a5c_0" }} + "#, + platform = Platform::current(), + ); + + let pixi = PixiControl::from_manifest(&manifest).expect("cannot instantiate pixi project"); + + temp_env::async_with_vars( + [( + "RATTLER_AUTH_FILE", + Some(credentials_path.to_str().unwrap()), + )], + async { + pixi.install().await.unwrap(); + }, + ) + .await; + + // Test for existence of conda-meta/my-webserver-0.1.0-pyh4616a5c_0.json file + assert!(pixi + .default_env_path() + .unwrap() + .join("conda-meta") + .join("my-webserver-0.1.0-pyh4616a5c_0.json") + .exists()); +} diff --git a/tests/integration_rust/solve_group_tests.rs b/tests/integration_rust/solve_group_tests.rs index 0ad901912..d2475e373 100644 --- a/tests/integration_rust/solve_group_tests.rs +++ b/tests/integration_rust/solve_group_tests.rs @@ -169,7 +169,7 @@ async fn test_purl_are_missing_for_non_conda_forge() { pixi.init().await.unwrap(); let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let foo_bar_package = Package::build("foo-bar-car", "2").finish(); let mut repo_data_record = RepoDataRecord { @@ -213,7 +213,7 @@ async fn test_purl_are_generated_using_custom_mapping() { pixi.init().await.unwrap(); let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let foo_bar_package = Package::build("foo-bar-car", "2").finish(); let mut repo_data_record = RepoDataRecord { @@ -258,7 +258,7 @@ async fn test_compressed_mapping_catch_not_pandoc_not_a_python_package() { pixi.init().await.unwrap(); let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let foo_bar_package = Package::build("pandoc", "2").finish(); let mut repo_data_record = RepoDataRecord { @@ -299,7 +299,7 @@ async fn test_dont_record_not_present_package_as_purl() { pixi.init().await.unwrap(); let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); // We use one package that is present in our mapping: `boltons` // and another one that is missing from conda and our mapping: // `pixi-something-new-for-test` because `pixi-something-new-for-test` is @@ -393,7 +393,7 @@ async fn test_we_record_not_present_package_as_purl_for_custom_mapping() { let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); // We use one package that is present in our mapping: `boltons` // and another one that is missing from conda and our mapping: @@ -479,7 +479,7 @@ async fn test_custom_mapping_channel_with_suffix() { let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let foo_bar_package = Package::build("pixi-something-new", "2").finish(); @@ -530,7 +530,7 @@ async fn test_repo_data_record_channel_with_suffix() { let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let foo_bar_package = Package::build("pixi-something-new", "2").finish(); @@ -581,7 +581,7 @@ async fn test_path_channel() { let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let foo_bar_package = Package::build("pixi-something-new", "2").finish(); @@ -655,7 +655,7 @@ async fn test_file_url_as_mapping_location() { let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let foo_bar_package = Package::build("pixi-something-new", "2").finish(); @@ -706,7 +706,7 @@ async fn test_disabled_mapping() { let project = pixi.workspace().unwrap(); - let client = project.authenticated_client(); + let client = project.authenticated_client().unwrap(); let blocking_middleware = OfflineMiddleware;