Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: cache nix commands #1468

Merged
merged 52 commits into from
Oct 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
6b0bff2
devenv: integrate nix command caching
sandydoo Sep 20, 2024
e914a6c
devenv: fix cached command path normalization
sandydoo Sep 20, 2024
c1f706b
nix-cache: add verbosity and re-org the command cacher
sandydoo Sep 24, 2024
a3f721d
nix-cache: add callback for stderr
sandydoo Sep 24, 2024
26233ed
nix-cache: handle removed files
sandydoo Sep 24, 2024
1731fc7
chore: unnest code
sandydoo Sep 24, 2024
27dfba9
nix-cache: rename tables
sandydoo Sep 24, 2024
a9164eb
nix-cache: delete previous commands
sandydoo Sep 24, 2024
a4da14f
nix-cache: add explicit uints to verbosity variants
sandydoo Sep 24, 2024
c8aade4
nix-cache: drop nix prefix and add result and activity types
sandydoo Sep 26, 2024
4b4d0b6
nix-cache: drop debug prints
sandydoo Sep 26, 2024
596b245
nix-cache: watch devenv.yaml by default
sandydoo Sep 26, 2024
e171283
cli: add options to disable and refresh the eval cache
sandydoo Sep 26, 2024
e7441cc
cnix: disable the flake eval cache by default
sandydoo Sep 26, 2024
eb02641
chore: rename nix-cache to devenv-eval-cache
sandydoo Sep 26, 2024
40ca67b
chore: use local packages from workspace
sandydoo Sep 26, 2024
cde7636
cnix: figure out logging
sandydoo Sep 26, 2024
ac5e33a
chore: fix warnings
sandydoo Sep 26, 2024
c5b4efe
eval-cache: update file timestamp
sandydoo Sep 26, 2024
d34a989
eval-cache: remove unused json output option
sandydoo Sep 26, 2024
b86baeb
chrore: add missing task option descriptions
sandydoo Sep 26, 2024
b974835
chore: lint
sandydoo Sep 26, 2024
73fd452
cnix: opt in commands for caching
sandydoo Sep 26, 2024
2bf7b74
chore: docs
sandydoo Sep 26, 2024
b5daa5c
cli: fix no-prefixed bool negation flags
sandydoo Sep 26, 2024
0c58894
chore: make cli module public
sandydoo Sep 26, 2024
96f27d4
devenv: create dirs before initialising cnix
sandydoo Sep 27, 2024
9d91b23
Auto generate docs/reference/options.md
github-actions[bot] Sep 27, 2024
f784142
chore: lint
sandydoo Sep 27, 2024
5f8e0bc
devenv: optimize and strip
sandydoo Sep 27, 2024
10c3b50
chore: lint
sandydoo Sep 27, 2024
fe08260
devenv: support writing out command deps to file for direnv to watch
sandydoo Sep 27, 2024
cce56a5
direnvrc: watch eval file dependencies
sandydoo Sep 27, 2024
9d9dfc4
add blog post
domenkozar Sep 30, 2024
66fe1c7
eval-cache: add way to exclude files from being tracked
sandydoo Sep 29, 2024
d4585d3
eval-cache: track modified time and last run timestamp
sandydoo Sep 29, 2024
e247aff
eval-cache: store hash content hashes to detect file changes between …
sandydoo Sep 30, 2024
790c423
chore: add docs and clean up
sandydoo Sep 30, 2024
6f02d10
blog: wip caching post
sandydoo Oct 1, 2024
05aacdf
eval-cache: add modification detection tests
sandydoo Oct 1, 2024
36bdc0c
eval-cache: add error log detection
sandydoo Oct 1, 2024
c00bbf4
eval-cache: add readDir support
sandydoo Oct 1, 2024
2200f4a
eval-cache: sort by path to keep input hash the same
sandydoo Oct 1, 2024
d0be300
eval-cache: fix hash computation for directories
sandydoo Oct 1, 2024
137da9a
eval-cache: fix builtins detection for patched nix
sandydoo Oct 1, 2024
40fd0d5
eval-cache: dedup tracked paths
sandydoo Oct 1, 2024
d974578
eval-cache: ignore dotfile folder
sandydoo Oct 1, 2024
c65330e
devenv: keep creating imports.txt for backwards-compat
sandydoo Oct 1, 2024
2bd0bad
eval-cache: ignore missing files
sandydoo Oct 1, 2024
09d1534
blog: update 1.3
sandydoo Oct 2, 2024
5a70ac1
eval-cache: expose stderr for unlogged command errors
sandydoo Oct 2, 2024
5a32581
mark blog post as draft
domenkozar Oct 2, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
973 changes: 944 additions & 29 deletions Cargo.lock

Large diffs are not rendered by default.

22 changes: 21 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[workspace]
resolver = "2"
members = ["devenv", "devenv-run-tests", "xtask", "tasks"]
members = ["devenv", "devenv-eval-cache", "devenv-run-tests", "tasks", "xtask"]

[workspace.package]
edition = "2021"
Expand All @@ -9,14 +9,23 @@ homepage = "https://devenv.sh/"
repository = "https://github.com/cachix/devenv/"

[workspace.dependencies]
devenv = { path = "devenv" }
devenv-eval-cache = { path = "devenv-eval-cache" }
devenv-run-tests = { path = "devenv-run-tests" }
tasks = { path = "tasks" }
xtask = { path = "xtask" }

ansiterm = "0.12.2"
blake3 = "1.5.4"
clap = { version = "4.5.1", features = ["derive", "cargo"] }
cli-table = "0.4.7"
dotlock = "0.5.0"
fs2 = "0.4.3"
futures = "0.3.30"
hex = "0.4.3"
include_dir = "0.7.3"
indoc = "2.0.4"
lazy_static = "1.5.0"
miette = { version = "7.1.0", features = ["fancy"] }
nix = { version = "0.28.0", features = ["signal"] }
regex = "1.10.3"
Expand All @@ -29,12 +38,23 @@ schematic = { version = "0.14.3", features = [
] }
serde = "1.0.197"
serde_json = "1.0.114"
serde_repr = "0.1.19"
serde_yaml = "0.9.32"
sha2 = "0.10.8"
sqlx = { version = "0.8.2", features = ["time", "sqlite", "runtime-tokio"] }
tempdir = "0.3.7"
thiserror = "1.0.63"
tracing = "0.1.40"
which = "6.0.0"
whoami = "1.5.1"
xdg = "2.5.2"
tokio = "1.39.3"
schemars = "0.8.16"

# Always build optimized sqlx-macro to speed up query checks
[profile.dev.package.sqlx-macros]
opt-level = 3

[profile.release]
strip = true
lto = "fat"
22 changes: 22 additions & 0 deletions devenv-eval-cache/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[package]
name = "devenv-eval-cache"
# TODO: switch to workspace version
version = "0.1.0"
edition.workspace = true
license.workspace = true

[dependencies]
blake3.workspace = true
futures.workspace = true
lazy_static.workspace = true
miette.workspace = true
regex.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_repr.workspace = true
sqlx.workspace = true
thiserror.workspace = true
tokio.workspace = true

[dev-dependencies]
tempdir.workspace = true
43 changes: 43 additions & 0 deletions devenv-eval-cache/migrations/20240906130404_init.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
CREATE TABLE IF NOT EXISTS cached_cmd
(
id INTEGER NOT NULL PRIMARY KEY,
raw TEXT NOT NULL,
cmd_hash CHAR(64) NOT NULL UNIQUE,
input_hash CHAR(64) NOT NULL,
output TEXT NOT NULL,
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
);

CREATE INDEX IF NOT EXISTS idx_cached_cmd_hash ON cached_cmd(cmd_hash);

CREATE TABLE IF NOT EXISTS file_path
(
id INTEGER NOT NULL PRIMARY KEY,
path BLOB NOT NULL UNIQUE,
is_directory BOOLEAN NOT NULL,
content_hash CHAR(64) NOT NULL,
modified_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
);

CREATE INDEX IF NOT EXISTS idx_file_path ON file_path(path);

CREATE TABLE IF NOT EXISTS cmd_input_path
(
id INTEGER NOT NULL PRIMARY KEY,
cached_cmd_id INTEGER,
file_path_id INTEGER,
UNIQUE(cached_cmd_id, file_path_id),
FOREIGN KEY(cached_cmd_id)
REFERENCES cached_cmd(id)
ON UPDATE CASCADE
ON DELETE CASCADE,
FOREIGN KEY(file_path_id)
REFERENCES file_path(id)
ON UPDATE CASCADE
ON DELETE CASCADE
);

CREATE INDEX IF NOT EXISTS idx_cmd_input_path_cached_cmd_id ON cmd_input_path(cached_cmd_id);
CREATE INDEX IF NOT EXISTS idx_cmd_input_path_file_path_id ON cmd_input_path(file_path_id);
CREATE INDEX IF NOT EXISTS idx_cmd_input_path_composite ON cmd_input_path(cached_cmd_id, file_path_id);
17 changes: 17 additions & 0 deletions devenv-eval-cache/src/bin/main.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
use std::process::Command;

use devenv_eval_cache::{command, db};

#[tokio::main]
async fn main() -> Result<(), command::CommandError> {
let database_url = "sqlite:nix-eval-cache.db";
let pool = db::setup_db(database_url).await?;

let mut cmd = Command::new("nix");
cmd.args(["eval", ".#devenv.processes"]);

let output = command::CachedCommand::new(&pool).output(&mut cmd).await?;
println!("{}", String::from_utf8_lossy(&output.stdout));

Ok(())
}
Loading
Loading