Cannot import `pkgs.glibc` in flake with devenv #486

aaronmondal · 2023-03-13T14:33:16Z

Describe the bug
While porting the Nix flake in rules_ll to devenv, I found that pkgs.glibc in the packages section will lead to nix develop hanging indefinitely on startup.

Seems depending on glibc somehow causes binaries to use an incompatible version. In my case I think this can be worked around by adding paths to glibc headers and library in scripts manually and passing that as environment variables, but the current behavior doesn't seem like it's WAI.

To Reproduce

{
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11";
    devenv.url = "github:cachix/devenv";
  };

  outputs = { self, nixpkgs, devenv, ... } @ inputs:
    let
      systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
      forAllSystems = f: builtins.listToAttrs (map (name: { inherit name; value = f name; }) systems);
    in
    {
      devShells = forAllSystems
        (system:
          let
            pkgs = import nixpkgs {
              inherit system;
            };
          in
          {
            default = devenv.lib.mkShell {
              inherit inputs pkgs;
              modules = [
                {
                  # https://devenv.sh/reference/options/
                  packages = [
                    # Base dependencies.
                    pkgs.hello
                    pkgs.glibc
                  ];

                  enterShell = ''
                  hello
                  '';
                }
              ];
            };
          });
    };
}

flake.lock

{
  "nodes": {
    "devenv": {
      "inputs": {
        "flake-compat": "flake-compat",
        "nix": "nix",
        "nixpkgs": "nixpkgs",
        "pre-commit-hooks": "pre-commit-hooks"
      },
      "locked": {
        "lastModified": 1678717095,
        "narHash": "sha256-fRUvPhdYNd/B15oKEdqKSavmOyDjXHpGJK+dlSwZWQQ=",
        "owner": "cachix",
        "repo": "devenv",
        "rev": "b331906925f3ba23d67e461b14ed042c743259c4",
        "type": "github"
      },
      "original": {
        "owner": "cachix",
        "repo": "devenv",
        "type": "github"
      }
    },
    "flake-compat": {
      "flake": false,
      "locked": {
        "lastModified": 1673956053,
        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
        "owner": "edolstra",
        "repo": "flake-compat",
        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
        "type": "github"
      },
      "original": {
        "owner": "edolstra",
        "repo": "flake-compat",
        "type": "github"
      }
    },
    "flake-utils": {
      "locked": {
        "lastModified": 1667395993,
        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
        "owner": "numtide",
        "repo": "flake-utils",
        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "flake-utils",
        "type": "github"
      }
    },
    "gitignore": {
      "inputs": {
        "nixpkgs": [
          "devenv",
          "pre-commit-hooks",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1660459072,
        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
        "owner": "hercules-ci",
        "repo": "gitignore.nix",
        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
        "type": "github"
      },
      "original": {
        "owner": "hercules-ci",
        "repo": "gitignore.nix",
        "type": "github"
      }
    },
    "lowdown-src": {
      "flake": false,
      "locked": {
        "lastModified": 1633514407,
        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
        "owner": "kristapsdz",
        "repo": "lowdown",
        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
        "type": "github"
      },
      "original": {
        "owner": "kristapsdz",
        "repo": "lowdown",
        "type": "github"
      }
    },
    "nix": {
      "inputs": {
        "lowdown-src": "lowdown-src",
        "nixpkgs": [
          "devenv",
          "nixpkgs"
        ],
        "nixpkgs-regression": "nixpkgs-regression"
      },
      "locked": {
        "lastModified": 1676545802,
        "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=",
        "owner": "domenkozar",
        "repo": "nix",
        "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f",
        "type": "github"
      },
      "original": {
        "owner": "domenkozar",
        "ref": "relaxed-flakes",
        "repo": "nix",
        "type": "github"
      }
    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1678500213,
        "narHash": "sha256-A5s2rXawJ+dCThkMXoMuYW8dgyUmkElcyfVJUot/Vr0=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "2ce9b9842b5e63884dfc3dea6689769e2a1ea309",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "ref": "nixpkgs-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs-regression": {
      "locked": {
        "lastModified": 1643052045,
        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
        "type": "github"
      }
    },
    "nixpkgs-stable": {
      "locked": {
        "lastModified": 1673800717,
        "narHash": "sha256-SFHraUqLSu5cC6IxTprex/nTsI81ZQAtDvlBvGDWfnA=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "2f9fd351ec37f5d479556cd48be4ca340da59b8f",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "ref": "nixos-22.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs_2": {
      "locked": {
        "lastModified": 1678614274,
        "narHash": "sha256-kK9l45n9cQ4mEQeT4NvJ+pK2nwsHhfZGUyRgzI+XFOw=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "970402e6147c49603f4d06defe44d27fe51884ce",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "ref": "nixos-22.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "pre-commit-hooks": {
      "inputs": {
        "flake-compat": [
          "devenv",
          "flake-compat"
        ],
        "flake-utils": "flake-utils",
        "gitignore": "gitignore",
        "nixpkgs": [
          "devenv",
          "nixpkgs"
        ],
        "nixpkgs-stable": "nixpkgs-stable"
      },
      "locked": {
        "lastModified": 1678376203,
        "narHash": "sha256-3tyYGyC8h7fBwncLZy5nCUjTJPrHbmNwp47LlNLOHSM=",
        "owner": "cachix",
        "repo": "pre-commit-hooks.nix",
        "rev": "1a20b9708962096ec2481eeb2ddca29ed747770a",
        "type": "github"
      },
      "original": {
        "owner": "cachix",
        "repo": "pre-commit-hooks.nix",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
        "devenv": "devenv",
        "nixpkgs": "nixpkgs_2"
      }
    }
  },
  "root": "root",
  "version": 7
}

Version

nix version 2.13.3
In case it matters: my host glibc is 2.36-r7 p9 from Gentoo which is incompatible with glibc-2.35-224 from nix.

The text was updated successfully, but these errors were encountered:

thenonameguy · 2023-03-17T09:36:54Z

@aaronmondal what happens if you try adding pkgs.glibc.out?

aaronmondal · 2023-03-21T12:17:56Z

@thenonameguy Same issue. I'm not entirely sure why, but several hours after experimenting a bit with pkgs.glibc and pkgs.glibc.out my system crashed for the first time in like forever (overnight, can't reproduce though :/).

Maybe that was just a coincidence though. Is there a chance this is leaking memory somehow in a way that triggers the kernel to force a shut down?

thenonameguy · 2023-03-21T14:06:07Z

@thenonameguy Same issue. I'm not entirely sure why, but several hours after experimenting a bit with pkgs.glibc and pkgs.glibc.out my system crashed for the first time in like forever (overnight, can't reproduce though :/).

Maybe that was just a coincidence though. Is there a chance this is leaking memory somehow in a way that triggers the kernel to force a shut down?

The only overnight running process (that can leak memory continuously) is the nix-daemon process running as root.
Since you are on the latest stable version, I think the chances are that this is not the root cause.
You did a multi-user Nix install, right?

aaronmondal · 2023-03-21T14:49:02Z

You did a multi-user Nix install, right?

Yes. Via the sh <(curl -L https://nixos.org/nix/install) --daemon command from the docs. That's working fine for everything else. It's only glibc that doesn't work and only when using devenv. In a regular mkShell things work fine.

aaronmondal · 2023-03-26T15:41:33Z

@thenonameguy Random guess: Could it be that devenv uses ldconfig differently than mkShell so that the dynamic loader falls back to the host's ld.so.conf and loads the host's glibc during runtime? Or maybe the host's ld.so.conf is read before the one generated in nixpkgs.glibc(.out), causing the host glibc to override the one fron nix?

thenonameguy · 2023-03-26T15:48:06Z

Interesting idea!
I have branch that reverts the naked shell change and uses a subset of the mkShell env vars:
thenonameguy:feat/stripped-shell

Can you try setting this as your devenv.yaml:

inputs:
  nixpkgs:
    url: github:NixOS/nixpkgs/nixpkgs-unstable
  devenv:
    url: github:thenonameguy/devenv/feat/stripped-shell

And seeing if the problem persists? @aaronmondal

aaronmondal · 2023-03-26T16:48:19Z

Ahh yes this seems to work! Using your fork appears to make things work. So I think #507 will fix this issue.

sandydoo · 2023-03-29T21:51:39Z

The issue here is that the naked shell is setting LD_LIBRARY_PATH to a path containing libraries for the requested packages. If that path happens to contain a runtime dependency of a binary executed in the shell, it'll be loaded instead of the library hardcoded into the binary with rpath. In this case, nix is loading a version of glibc that's not compatible with what it was compiled with. You'll either get an error about missing symbols or something much, much worse.

This affects all binaries with runtime dependencies built with nix. The solution is to,

Not mess around with LD_LIBRARY_PATH
Restore setup hooks

, which is what #507 does by using mkShell.

Fun fact, Domen and I have somehow run into the library path issue three times in the last week, all in entirely different contexts 😂

cc @domenkozar

domenkozar · 2023-08-01T15:21:33Z

Can you try #745

Nick1296 · 2023-09-18T22:29:18Z

Just tried to import pkgs.glibc using #745, I am getting the same error as with the latest stable:

symbol lookup error: /nix/store/q52d39as1wr36sg3lavmwr1wvi212kps-devenv-profile/lib/libc.so.6: undefined symbol: _dl_audit_symbind_alt, version GLIBC_PRIVATE

I am happy to privde more info on this issue if needed and thanks for the amazing work!

domenkozar · 2023-09-19T09:05:08Z

@Nick1296 could you provide more information on what OS/platform are you? And the exact devenv.nix you used.

Nick1296 · 2023-09-19T15:47:58Z

I tried importing glibc with devenv 0.6.3 on NixOs 23.11 (where I get a segfault).
I tried also with devenv 0.6.3 on fedora 32 and I was getting a segfault. So I decided to try the version proposed on #745 and I got the error in my previous comment.

I am happy to do more tests if you need me to.

aaronmondal added the bug Something isn't working label Mar 13, 2023

aaronmondal mentioned this issue Mar 26, 2023

feat(shell): revert from naked shell to pkgs.mkShell + unsetting irrelevant env vars #507

Closed

domenkozar mentioned this issue Aug 1, 2023

1.0: Rewrite in Python #745

Closed

7 tasks

domenkozar mentioned this issue Mar 11, 2024

Rust rewrite #1005

Merged

2 tasks

domenkozar closed this as completed in #1005 Mar 20, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Cannot import `pkgs.glibc` in flake with devenv #486

Cannot import `pkgs.glibc` in flake with devenv #486

aaronmondal commented Mar 13, 2023

thenonameguy commented Mar 17, 2023

aaronmondal commented Mar 21, 2023 •

edited

Loading

thenonameguy commented Mar 21, 2023

aaronmondal commented Mar 21, 2023 •

edited

Loading

aaronmondal commented Mar 26, 2023

thenonameguy commented Mar 26, 2023 •

edited

Loading

aaronmondal commented Mar 26, 2023 •

edited

Loading

sandydoo commented Mar 29, 2023

domenkozar commented Aug 1, 2023

Nick1296 commented Sep 18, 2023

domenkozar commented Sep 19, 2023

Nick1296 commented Sep 19, 2023

Cannot import pkgs.glibc in flake with devenv #486

Cannot import pkgs.glibc in flake with devenv #486

Comments

aaronmondal commented Mar 13, 2023

thenonameguy commented Mar 17, 2023

aaronmondal commented Mar 21, 2023 • edited Loading

thenonameguy commented Mar 21, 2023

aaronmondal commented Mar 21, 2023 • edited Loading

aaronmondal commented Mar 26, 2023

thenonameguy commented Mar 26, 2023 • edited Loading

aaronmondal commented Mar 26, 2023 • edited Loading

sandydoo commented Mar 29, 2023

domenkozar commented Aug 1, 2023

Nick1296 commented Sep 18, 2023

domenkozar commented Sep 19, 2023

Nick1296 commented Sep 19, 2023

Cannot import `pkgs.glibc` in flake with devenv #486

Cannot import `pkgs.glibc` in flake with devenv #486

aaronmondal commented Mar 21, 2023 •

edited

Loading

aaronmondal commented Mar 21, 2023 •

edited

Loading

thenonameguy commented Mar 26, 2023 •

edited

Loading

aaronmondal commented Mar 26, 2023 •

edited

Loading