From 7c202e3efc8f3f5784778ff06ddb359db923016a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Fri, 19 Jan 2024 12:56:44 +0000 Subject: [PATCH] ceph: Fix tests by adding back old required python libs. See #281858. Fixes #241482. Also fix test putting cluster in unhealthy `POOL_APP_NOT_ENABLED` state; this seems to be the default state with Ceph 18. --- nixos/tests/ceph-multi-node.nix | 8 ++ nixos/tests/ceph-single-node-bluestore.nix | 8 ++ nixos/tests/ceph-single-node.nix | 8 ++ .../python-modules/cryptography/40.nix | 112 ++++++++++++++++++ .../cryptography/40_vectors.nix | 27 +++++ .../python-modules/kubernetes/18.nix | 63 ++++++++++ .../python-modules/pyopenssl/23_1.nix | 100 ++++++++++++++++ pkgs/tools/filesystems/ceph/default.nix | 12 ++ pkgs/top-level/python-packages.nix | 16 +++ 9 files changed, 354 insertions(+) create mode 100644 pkgs/development/python-modules/cryptography/40.nix create mode 100644 pkgs/development/python-modules/cryptography/40_vectors.nix create mode 100644 pkgs/development/python-modules/kubernetes/18.nix create mode 100644 pkgs/development/python-modules/pyopenssl/23_1.nix diff --git a/nixos/tests/ceph-multi-node.nix b/nixos/tests/ceph-multi-node.nix index 556546beee764bd..b1352a4bc8f4ecc 100644 --- a/nixos/tests/ceph-multi-node.nix +++ b/nixos/tests/ceph-multi-node.nix @@ -185,6 +185,14 @@ let monA.succeed( "ceph osd pool create multi-node-test 32 32", "ceph osd pool ls | grep 'multi-node-test'", + + # We need to enable an application on the pool, otherwise it will + # stay unhealthy in state POOL_APP_NOT_ENABLED. + # Creating a CephFS would do this automatically, but we haven't done that here. + # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application + # We use the custom application name "nixos-test" for this. + "ceph osd pool application enable multi-node-test nixos-test", + "ceph osd pool rename multi-node-test multi-node-other-test", "ceph osd pool ls | grep 'multi-node-other-test'", ) diff --git a/nixos/tests/ceph-single-node-bluestore.nix b/nixos/tests/ceph-single-node-bluestore.nix index acaae4cf300e891..8bd1a78244a291a 100644 --- a/nixos/tests/ceph-single-node-bluestore.nix +++ b/nixos/tests/ceph-single-node-bluestore.nix @@ -145,6 +145,14 @@ let monA.succeed( "ceph osd pool create single-node-test 32 32", "ceph osd pool ls | grep 'single-node-test'", + + # We need to enable an application on the pool, otherwise it will + # stay unhealthy in state POOL_APP_NOT_ENABLED. + # Creating a CephFS would do this automatically, but we haven't done that here. + # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application + # We use the custom application name "nixos-test" for this. + "ceph osd pool application enable single-node-test nixos-test", + "ceph osd pool rename single-node-test single-node-other-test", "ceph osd pool ls | grep 'single-node-other-test'", ) diff --git a/nixos/tests/ceph-single-node.nix b/nixos/tests/ceph-single-node.nix index a3a4072365af821..fdc065583e1c08d 100644 --- a/nixos/tests/ceph-single-node.nix +++ b/nixos/tests/ceph-single-node.nix @@ -145,6 +145,14 @@ let monA.succeed( "ceph osd pool create single-node-test 32 32", "ceph osd pool ls | grep 'single-node-test'", + + # We need to enable an application on the pool, otherwise it will + # stay unhealthy in state POOL_APP_NOT_ENABLED. + # Creating a CephFS would do this automatically, but we haven't done that here. + # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application + # We use the custom application name "nixos-test" for this. + "ceph osd pool application enable single-node-test nixos-test", + "ceph osd pool rename single-node-test single-node-other-test", "ceph osd pool ls | grep 'single-node-other-test'", ) diff --git a/pkgs/development/python-modules/cryptography/40.nix b/pkgs/development/python-modules/cryptography/40.nix new file mode 100644 index 000000000000000..34d1c69b184e457 --- /dev/null +++ b/pkgs/development/python-modules/cryptography/40.nix @@ -0,0 +1,112 @@ +{ lib +, stdenv +, callPackage +, buildPythonPackage +, fetchPypi +, rustPlatform +, cargo +, rustc +, setuptoolsRustBuildHook +, openssl +, Security +, isPyPy +, cffi +, pkg-config +, pytestCheckHook +, pytest-subtests +, pythonOlder +, pretend +, libiconv +, libxcrypt +, iso8601 +, py +, pytz +, hypothesis +}: + +let + cryptography-vectors = callPackage ./40_vectors.nix { }; +in +buildPythonPackage rec { + pname = "cryptography"; + version = "40.0.1"; # Also update the hash in vectors.nix + format = "setuptools"; + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI="; + }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + sourceRoot = "${pname}-${version}/${cargoRoot}"; + name = "${pname}-${version}"; + hash = "sha256-gFfDTc2QWBWHBCycVH1dYlCsWQMVcRZfOBIau+njtDU="; + }; + + postPatch = '' + substituteInPlace pyproject.toml \ + --replace "--benchmark-disable" "" + ''; + + cargoRoot = "src/rust"; + + nativeBuildInputs = [ + rustPlatform.cargoSetupHook + setuptoolsRustBuildHook + cargo + rustc + pkg-config + ] ++ lib.optionals (!isPyPy) [ + cffi + ]; + + buildInputs = [ openssl ] + ++ lib.optionals stdenv.isDarwin [ Security libiconv ] + ++ lib.optionals (pythonOlder "3.9") [ libxcrypt ]; + + propagatedBuildInputs = lib.optionals (!isPyPy) [ + cffi + ]; + + nativeCheckInputs = [ + cryptography-vectors + hypothesis + iso8601 + pretend + py + pytestCheckHook + pytest-subtests + pytz + ]; + + pytestFlagsArray = [ + "--disable-pytest-warnings" + ]; + + disabledTestPaths = [ + # save compute time by not running benchmarks + "tests/bench" + ] ++ lib.optionals (stdenv.isDarwin && stdenv.isAarch64) [ + # aarch64-darwin forbids W+X memory, but this tests depends on it: + # * https://cffi.readthedocs.io/en/latest/using.html#callbacks + "tests/hazmat/backends/test_openssl_memleak.py" + ]; + + meta = with lib; { + description = "A package which provides cryptographic recipes and primitives"; + longDescription = '' + Cryptography includes both high level recipes and low level interfaces to + common cryptographic algorithms such as symmetric ciphers, message + digests, and key derivation functions. + Our goal is for it to be your "cryptographic standard library". It + supports Python 2.7, Python 3.5+, and PyPy 5.4+. + ''; + homepage = "https://github.com/pyca/cryptography"; + changelog = "https://cryptography.io/en/latest/changelog/#v" + + replaceStrings [ "." ] [ "-" ] version; + license = with licenses; [ asl20 bsd3 psfl ]; + maintainers = with maintainers; [ SuperSandro2000 ]; + }; +} diff --git a/pkgs/development/python-modules/cryptography/40_vectors.nix b/pkgs/development/python-modules/cryptography/40_vectors.nix new file mode 100644 index 000000000000000..096865075492d7a --- /dev/null +++ b/pkgs/development/python-modules/cryptography/40_vectors.nix @@ -0,0 +1,27 @@ +{ buildPythonPackage, fetchPypi, lib, cryptography_40 }: + +buildPythonPackage rec { + pname = "cryptography-vectors"; + # The test vectors must have the same version as the cryptography package + inherit (cryptography_40) version; + format = "setuptools"; + + src = fetchPypi { + pname = "cryptography_vectors"; + inherit version; + hash = "sha256-hGBwa1tdDOSoVXHKM4nPiPcAu2oMYTPcn+D1ovW9oEE="; + }; + + # No tests included + doCheck = false; + + pythonImportsCheck = [ "cryptography_vectors" ]; + + meta = with lib; { + description = "Test vectors for the cryptography package"; + homepage = "https://cryptography.io/en/latest/development/test-vectors/"; + # Source: https://github.com/pyca/cryptography/tree/master/vectors; + license = with licenses; [ asl20 bsd3 ]; + maintainers = with maintainers; [ SuperSandro2000 ]; + }; +} diff --git a/pkgs/development/python-modules/kubernetes/18.nix b/pkgs/development/python-modules/kubernetes/18.nix new file mode 100644 index 000000000000000..0f7cb7277f56b12 --- /dev/null +++ b/pkgs/development/python-modules/kubernetes/18.nix @@ -0,0 +1,63 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, pythonOlder + +# propgatedBuildInputs +, adal +, certifi +, google-auth +, python-dateutil +, pyyaml +, requests +, requests-oauthlib +, urllib3 +, websocket-client + +# tests +, pytestCheckHook +, mock +}: + +buildPythonPackage rec { + pname = "kubernetes"; + version = "18.20.0"; + format = "setuptools"; + disabled = pythonOlder "3.6"; + + src = fetchFromGitHub { + owner = "kubernetes-client"; + repo = "python"; + rev = "v${version}"; + sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1"; + fetchSubmodules = true; + }; + + propagatedBuildInputs = [ + adal + certifi + google-auth + python-dateutil + pyyaml + requests + requests-oauthlib + urllib3 + websocket-client + ]; + + pythonImportsCheck = [ + "kubernetes" + ]; + + checkInputs = [ + mock + pytestCheckHook + ]; + + meta = with lib; { + description = "Kubernetes python client"; + homepage = "https://github.com/kubernetes-client/python"; + license = licenses.asl20; + maintainers = with maintainers; [ lsix ]; + }; +} diff --git a/pkgs/development/python-modules/pyopenssl/23_1.nix b/pkgs/development/python-modules/pyopenssl/23_1.nix new file mode 100644 index 000000000000000..db77f854dbb3081 --- /dev/null +++ b/pkgs/development/python-modules/pyopenssl/23_1.nix @@ -0,0 +1,100 @@ +{ lib +, stdenv +, buildPythonPackage +, fetchPypi +, openssl +, cryptography +, pytestCheckHook +, pretend +, sphinxHook +, sphinx-rtd-theme +, flaky +}: + +buildPythonPackage rec { + pname = "pyopenssl"; + version = "23.1.1"; + format = "setuptools"; + + src = fetchPypi { + pname = "pyOpenSSL"; + inherit version; + hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc="; + }; + + outputs = [ + "out" + "dev" + "doc" + ]; + + nativeBuildInputs = [ + openssl + sphinxHook + sphinx-rtd-theme + ]; + + postPatch = '' + # remove cryptography pin + sed "/cryptography/ s/,<[0-9]*//g" setup.py + ''; + + propagatedBuildInputs = [ + cryptography + ]; + + nativeCheckInputs = [ + flaky + pretend + pytestCheckHook + ]; + + __darwinAllowLocalNetworking = true; + + preCheck = '' + export LANG="en_US.UTF-8" + ''; + + disabledTests = [ + # https://github.com/pyca/pyopenssl/issues/692 + # These tests, we disable always. + "test_set_default_verify_paths" + "test_fallback_default_verify_paths" + # https://github.com/pyca/pyopenssl/issues/768 + "test_wantWriteError" + # https://github.com/pyca/pyopenssl/issues/1043 + "test_alpn_call_failure" + ] ++ lib.optionals (lib.hasPrefix "libressl" openssl.meta.name) [ + # https://github.com/pyca/pyopenssl/issues/791 + # These tests, we disable in the case that libressl is passed in as openssl. + "test_op_no_compression" + "test_npn_advertise_error" + "test_npn_select_error" + "test_npn_client_fail" + "test_npn_success" + "test_use_certificate_chain_file_unicode" + "test_use_certificate_chain_file_bytes" + "test_add_extra_chain_cert" + "test_set_session_id_fail" + "test_verify_with_revoked" + "test_set_notAfter" + "test_set_notBefore" + ] ++ lib.optionals (lib.versionAtLeast (lib.getVersion openssl.name) "1.1") [ + # these tests are extremely tightly wed to the exact output of the openssl cli tool, including exact punctuation. + "test_dump_certificate" + "test_dump_privatekey_text" + "test_dump_certificate_request" + "test_export_text" + ] ++ lib.optionals stdenv.is32bit [ + # https://github.com/pyca/pyopenssl/issues/974 + "test_verify_with_time" + ]; + + meta = with lib; { + description = "Python wrapper around the OpenSSL library"; + homepage = "https://github.com/pyca/pyopenssl"; + changelog = "https://github.com/pyca/pyopenssl/blob/${version}/CHANGELOG.rst"; + license = licenses.asl20; + maintainers = with maintainers; [ ]; + }; +} diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index f38cd4be880c02b..ac7aaf51b215a8c 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -167,6 +167,18 @@ let # Watch out for python <> boost compatibility python = python310.override { packageOverrides = self: super: { + # Ceph does not support `cryptography` > 40 yet: + # https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602 + # We pin the older `cryptography_40` here; + # this also forces us to pin an older `pyopenssl` because the current one + # is not compatible with older `cryptography`, see: + # https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30 + cryptography = super.cryptography_40; + pyopenssl = super.pyopenssl_23_1; + + # Ceph does not support `kubernetes` >= 19, see: + # https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090 + kubernetes = super.kubernetes_18; }; }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 173d4da103b5b9a..4554e038afdf51f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2457,6 +2457,13 @@ self: super: with self; { inherit (pkgs.darwin.apple_sdk.frameworks) Security; }; + # Old version needed for `ceph`. + # See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602 + cryptography_40 = callPackage ../development/python-modules/cryptography/40.nix { + inherit (pkgs.darwin) libiconv; + inherit (pkgs.darwin.apple_sdk.frameworks) Security; + }; + cryptolyzer = callPackage ../development/python-modules/cryptolyzer { }; cryptoparser = callPackage ../development/python-modules/cryptoparser { }; @@ -6171,6 +6178,8 @@ self: super: with self; { krfzf-py = callPackage ../development/python-modules/krfzf-py { }; kubernetes = callPackage ../development/python-modules/kubernetes { }; + # Needed for `ceph`, see: https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090 + kubernetes_18 = callPackage ../development/python-modules/kubernetes/18.nix { }; kurbopy = callPackage ../development/python-modules/kurbopy { }; @@ -10916,6 +10925,13 @@ self: super: with self; { pyopenssl = callPackage ../development/python-modules/pyopenssl { }; + # Old version needed for `ceph`. + # This is the most recent version of `pyopenssl` that's still compatible with `cryptography_40`. + # See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602 + pyopenssl_23_1 = callPackage ../development/python-modules/pyopenssl/23_1.nix { + cryptography = cryptography_40; + }; + pyopenuv = callPackage ../development/python-modules/pyopenuv { }; pyopnsense = callPackage ../development/python-modules/pyopnsense { };