Update dependencies and improve pyo3

- Update all dependencies so that `cargo outdated -R` is satisfied. - Use declarative pyo3 modules. - The docs and the code disagreed on whether the auto compressor entry point is named `compress_largest_rooms`, `compress_state_events_table` or `compress_chunks_of_database`. Rename the entry point to `run_compression`, matching the manual compressor, and fix the docs. Leave `compress_largest_rooms` as another entry point to avoid breaking existing code. - Provide a default for `default_levels`. - Use `&str` instead of `String` in Python signatures where it simplifies the code or is necessary to include the default in the text signature. - Release the GIL while running the manual compressor. - Use `map_err` to simplify error conversion a bit. Signed-off-by: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
matrix-org · Aug 4, 2024 · 408b588 · 408b588
1 parent 40b660e
commit 408b588
Show file tree

Hide file tree

Showing 8 changed files with 493 additions and 700 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -20,7 +20,7 @@ postgres-openssl = "0.5.0"
 rand = "0.8.5"
 rayon = "1.7.0"
 string_cache = "0.8.7"
-env_logger = "0.10.0"
+env_logger = "0.11.2"
 log = "0.4.20"
 log-panics = "2.1.0"
 
@@ -38,16 +38,16 @@ features = ["cargo"]
 optional = true
 
 [dependencies.pyo3]
-version = "0.19.2"
+version = "0.22.2"
 features = ["extension-module"]
 optional = true
 
 [dependencies.pyo3-log]
-version = "0.8.3"
+version = "0.11.0"
 optional = true
 
 [dependencies.tikv-jemallocator]
-version = "0.5.4"
+version = "0.6.0"
 optional = true
 
 [features]

diff --git a/compressor_integration_tests/Cargo.toml b/compressor_integration_tests/Cargo.toml
@@ -14,7 +14,7 @@ postgres-openssl = "0.5.0"
 rand = "0.8.5"
 synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
 synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
-env_logger = "0.10.0"
+env_logger = "0.11.2"
 log = "0.4.20"
 
 [dependencies.state-map]

diff --git a/docs/python.md b/docs/python.md
@@ -32,10 +32,9 @@ This will install the relevant compressor tool into the activated virtual enviro
 ```python
 import synapse_auto_compressor
 
-synapse_auto_compressor.compress_state_events_table(
+synapse_auto_compressor.run_compression(
   db_url="postgresql://localhost/synapse",
   chunk_size=500,
-  default_levels="100,50,25",
   number_of_chunks=100
 )
 ```

diff --git a/src/lib.rs b/src/lib.rs
@@ -22,7 +22,7 @@
 
 use log::{info, warn};
 #[cfg(feature = "pyo3")]
-use pyo3::{exceptions, prelude::*};
+use pyo3::prelude::*;
 
 #[cfg(feature = "clap")]
 use clap::{crate_authors, crate_description, crate_name, crate_version, Arg, Command};
@@ -745,89 +745,92 @@ impl Config {
     }
 }
 
-/// Access point for python code
-///
-/// Default arguments are equivalent to using the command line tool
-/// No default's are provided for db_url or room_id since these arguments
-/// are compulsory (so that new() act's like parse_arguments())
-#[cfg(feature = "pyo3")]
-#[allow(clippy::too_many_arguments)]
-#[pyfunction]
-#[pyo3(signature = (
-    // db_url has no default
-    db_url,
-
-    // room_id has no default
-    room_id,
-
-    output_file = None,
-    min_state_group = None,
-    groups_to_compress = None,
-    min_saved_rows = None,
-    max_state_group = None,
-    level_sizes = String::from("100,50,25"),
-
-    // have this default to true as is much worse to not have it if you need it
-    // than to have it and not need it
-    transactions = true,
-
-    graphs = false,
-    commit_changes = false,
-    verify = true,
-))]
-fn run_compression(
-    db_url: String,
-    room_id: String,
-    output_file: Option<String>,
-    min_state_group: Option<i64>,
-    groups_to_compress: Option<i64>,
-    min_saved_rows: Option<i32>,
-    max_state_group: Option<i64>,
-    level_sizes: String,
-    transactions: bool,
-    graphs: bool,
-    commit_changes: bool,
-    verify: bool,
-) -> PyResult<()> {
-    let config = Config::new(
-        db_url,
-        room_id,
-        output_file,
-        min_state_group,
-        groups_to_compress,
-        min_saved_rows,
-        max_state_group,
-        level_sizes,
-        transactions,
-        graphs,
-        commit_changes,
-        verify,
-    );
-    match config {
-        Err(e) => Err(PyErr::new::<exceptions::PyException, _>(e)),
-        Ok(config) => {
-            run(config);
-            Ok(())
-        }
-    }
-}
-
 /// Python module - "import synapse_compress_state" to use
 #[cfg(feature = "pyo3")]
 #[pymodule]
-fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> {
-    let _ = pyo3_log::Logger::default()
-        // don't send out anything lower than a warning from other crates
-        .filter(log::LevelFilter::Warn)
-        // don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
-        // situations and provides better log messages
-        .filter_target("synapse_compress_state".to_owned(), log::LevelFilter::Debug)
-        .install();
-    // ensure any panics produce error messages in the log
-    log_panics::init();
-
-    m.add_function(wrap_pyfunction!(run_compression, m)?)?;
-    Ok(())
+mod synapse_compress_state {
+    use super::*;
+    use log::LevelFilter;
+    use pyo3::exceptions::PyException;
+
+    #[pymodule_init]
+    fn init(_m: &Bound<'_, PyModule>) -> PyResult<()> {
+        let _ = pyo3_log::Logger::default()
+            // don't send out anything lower than a warning from other crates
+            .filter(LevelFilter::Warn)
+            // don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
+            // situations and provides better log messages
+            .filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug)
+            .install();
+
+        // ensure any panics produce error messages in the log
+        log_panics::init();
+
+        Ok(())
+    }
+
+    /// Main entry point
+    ///
+    /// Default arguments are equivalent to using the command line tool.
+    ///
+    /// No defaults are provided for `db_url` and `room_id` since these
+    /// arguments are mandatory.
+    #[allow(clippy::too_many_arguments)]
+    #[pyfunction]
+    #[pyo3(signature = (
+        db_url, // has no default
+        room_id, // has no default
+        output_file = None,
+        min_state_group = None,
+        groups_to_compress = None,
+        min_saved_rows = None,
+        max_state_group = None,
+        level_sizes = "100,50,25",
+
+        // have this default to true as is much worse to not have it if you need it
+        // than to have it and not need it
+        transactions = true,
+
+        graphs = false,
+        commit_changes = false,
+        verify = true,
+    ))]
+    fn run_compression(
+        py: Python,
+        db_url: String,
+        room_id: String,
+        output_file: Option<String>,
+        min_state_group: Option<i64>,
+        groups_to_compress: Option<i64>,
+        min_saved_rows: Option<i32>,
+        max_state_group: Option<i64>,
+        level_sizes: &str,
+        transactions: bool,
+        graphs: bool,
+        commit_changes: bool,
+        verify: bool,
+    ) -> PyResult<()> {
+        let config = Config::new(
+            db_url,
+            room_id,
+            output_file,
+            min_state_group,
+            groups_to_compress,
+            min_saved_rows,
+            max_state_group,
+            level_sizes.into(),
+            transactions,
+            graphs,
+            commit_changes,
+            verify,
+        )
+        .map_err(PyErr::new::<PyException, _>)?;
+
+        // Stops the compressor from holding the GIL while running
+        py.allow_threads(|| run(config));
+
+        Ok(())
+    }
 }
 
 // TESTS START HERE

diff --git a/synapse_auto_compressor/Cargo.toml b/synapse_auto_compressor/Cargo.toml
@@ -15,7 +15,7 @@ postgres-openssl = "0.5.0"
 rand = "0.8.5"
 serial_test = "2.0.0"
 synapse_compress_state = { path = "../", features = ["no-progress-bars"], default-features = false }
-env_logger = "0.10.0"
+env_logger = "0.11.2"
 log = "0.4.20"
 log-panics = "2.1.0"
 anyhow = "1.0.75"
@@ -30,16 +30,16 @@ features = ["cargo"]
 optional = true
 
 [dependencies.pyo3]
-version = "0.19.2"
+version = "0.22.2"
 features = ["extension-module"]
 optional = true
 
 [dependencies.pyo3-log]
-version = "0.8.3"
+version = "0.11.0"
 optional = true
 
 [dependencies.tikv-jemallocator]
-version = "0.5.4"
+version = "0.6.0"
 optional = true
 
 [features]