Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update dependencies and improve pyo3 #147

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Update dependencies and improve pyo3 #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
872 changes: 327 additions & 545 deletions Cargo.lock
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ postgres-openssl = "0.5.0"
rand = "0.8.5"
rayon = "1.7.0"
string_cache = "0.8.7"
env_logger = "0.10.0"
env_logger = "0.11.2"
log = "0.4.20"
log-panics = "2.1.0"

Expand All @@ -38,16 +38,16 @@ features = ["cargo"]
optional = true

[dependencies.pyo3]
version = "0.19.2"
version = "0.22.2"
features = ["extension-module"]
optional = true

[dependencies.pyo3-log]
version = "0.8.3"
version = "0.11.0"
optional = true

[dependencies.tikv-jemallocator]
version = "0.5.4"
version = "0.6.0"
optional = true

[features]
Expand Down
2 changes: 1 addition & 1 deletion compressor_integration_tests/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ postgres-openssl = "0.5.0"
rand = "0.8.5"
synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
env_logger = "0.10.0"
env_logger = "0.11.2"
log = "0.4.20"

[dependencies.state-map]
Expand Down
3 changes: 1 addition & 2 deletions docs/python.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,9 @@ This will install the relevant compressor tool into the activated virtual enviro
```python
import synapse_auto_compressor

synapse_auto_compressor.compress_state_events_table(
synapse_auto_compressor.run_compression(
db_url="postgresql://localhost/synapse",
chunk_size=500,
default_levels="100,50,25",
number_of_chunks=100
)
```
Expand Down
165 changes: 84 additions & 81 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

use log::{info, warn};
#[cfg(feature = "pyo3")]
use pyo3::{exceptions, prelude::*};
use pyo3::prelude::*;

#[cfg(feature = "clap")]
use clap::{crate_authors, crate_description, crate_name, crate_version, Arg, Command};
Expand Down Expand Up @@ -745,89 +745,92 @@ impl Config {
}
}

/// Access point for python code
///
/// Default arguments are equivalent to using the command line tool
/// No default's are provided for db_url or room_id since these arguments
/// are compulsory (so that new() act's like parse_arguments())
#[cfg(feature = "pyo3")]
#[allow(clippy::too_many_arguments)]
#[pyfunction]
#[pyo3(signature = (
// db_url has no default
db_url,

// room_id has no default
room_id,

output_file = None,
min_state_group = None,
groups_to_compress = None,
min_saved_rows = None,
max_state_group = None,
level_sizes = String::from("100,50,25"),

// have this default to true as is much worse to not have it if you need it
// than to have it and not need it
transactions = true,

graphs = false,
commit_changes = false,
verify = true,
))]
fn run_compression(
db_url: String,
room_id: String,
output_file: Option<String>,
min_state_group: Option<i64>,
groups_to_compress: Option<i64>,
min_saved_rows: Option<i32>,
max_state_group: Option<i64>,
level_sizes: String,
transactions: bool,
graphs: bool,
commit_changes: bool,
verify: bool,
) -> PyResult<()> {
let config = Config::new(
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes,
transactions,
graphs,
commit_changes,
verify,
);
match config {
Err(e) => Err(PyErr::new::<exceptions::PyException, _>(e)),
Ok(config) => {
run(config);
Ok(())
}
}
}

/// Python module - "import synapse_compress_state" to use
#[cfg(feature = "pyo3")]
#[pymodule]
fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> {
let _ = pyo3_log::Logger::default()
// don't send out anything lower than a warning from other crates
.filter(log::LevelFilter::Warn)
// don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
// situations and provides better log messages
.filter_target("synapse_compress_state".to_owned(), log::LevelFilter::Debug)
.install();
// ensure any panics produce error messages in the log
log_panics::init();

m.add_function(wrap_pyfunction!(run_compression, m)?)?;
Ok(())
mod synapse_compress_state {
use super::*;
use log::LevelFilter;
use pyo3::exceptions::PyException;

#[pymodule_init]
fn init(_m: &Bound<'_, PyModule>) -> PyResult<()> {
let _ = pyo3_log::Logger::default()
// don't send out anything lower than a warning from other crates
.filter(LevelFilter::Warn)
// don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
// situations and provides better log messages
.filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug)
.install();

// ensure any panics produce error messages in the log
log_panics::init();

Ok(())
}

/// Main entry point
///
/// Default arguments are equivalent to using the command line tool.
///
/// No defaults are provided for `db_url` and `room_id` since these
/// arguments are mandatory.
#[allow(clippy::too_many_arguments)]
#[pyfunction]
#[pyo3(signature = (
db_url, // has no default
room_id, // has no default
output_file = None,
min_state_group = None,
groups_to_compress = None,
min_saved_rows = None,
max_state_group = None,
level_sizes = "100,50,25",

// have this default to true as is much worse to not have it if you need it
// than to have it and not need it
transactions = true,

graphs = false,
commit_changes = false,
verify = true,
))]
fn run_compression(
py: Python,
db_url: String,
room_id: String,
output_file: Option<String>,
min_state_group: Option<i64>,
groups_to_compress: Option<i64>,
min_saved_rows: Option<i32>,
max_state_group: Option<i64>,
level_sizes: &str,
transactions: bool,
graphs: bool,
commit_changes: bool,
verify: bool,
) -> PyResult<()> {
let config = Config::new(
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes.into(),
transactions,
graphs,
commit_changes,
verify,
)
.map_err(PyErr::new::<PyException, _>)?;

// Stops the compressor from holding the GIL while running
py.allow_threads(|| run(config));

Ok(())
}
}

// TESTS START HERE
Expand Down
8 changes: 4 additions & 4 deletions synapse_auto_compressor/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ postgres-openssl = "0.5.0"
rand = "0.8.5"
serial_test = "2.0.0"
synapse_compress_state = { path = "../", features = ["no-progress-bars"], default-features = false }
env_logger = "0.10.0"
env_logger = "0.11.2"
log = "0.4.20"
log-panics = "2.1.0"
anyhow = "1.0.75"
Expand All @@ -30,16 +30,16 @@ features = ["cargo"]
optional = true

[dependencies.pyo3]
version = "0.19.2"
version = "0.22.2"
features = ["extension-module"]
optional = true

[dependencies.pyo3-log]
version = "0.8.3"
version = "0.11.0"
optional = true

[dependencies.tikv-jemallocator]
version = "0.5.4"
version = "0.6.0"
optional = true

[features]
Expand Down
Loading
Loading