From b8bedf78a648df67e3047b3d4dd3653447093aa1 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Mon, 11 Sep 2023 12:55:23 +0200 Subject: [PATCH] fix: span names, log levels and messages (#1213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: span names, log levels and messages * fix: codegen feature * improve log output * Update common/src/claims.rs Co-authored-by: Oddbjørn Grødem <29732646+oddgrd@users.noreply.github.com> --------- Co-authored-by: Oddbjørn Grødem <29732646+oddgrd@users.noreply.github.com> --- Makefile | 1 - cargo-shuttle/src/lib.rs | 7 ++---- codegen/Cargo.toml | 1 - codegen/src/shuttle_main/mod.rs | 43 -------------------------------- common/src/claims.rs | 2 +- common/src/deployment.rs | 9 +++---- common/src/log.rs | 6 +---- deployer/src/deployment/mod.rs | 2 +- deployer/src/deployment/queue.rs | 38 +++++++++++++--------------- deployer/src/deployment/run.rs | 34 ++++++++++--------------- deployer/src/handlers/mod.rs | 32 ++++++++++++++++++------ logger/prepare.sh | 8 ------ proto/src/lib.rs | 8 +++--- runtime/Cargo.toml | 1 - runtime/src/alpha/mod.rs | 37 +++++++++++++++++++++++++++ service/src/builder.rs | 22 ++++++++-------- 16 files changed, 115 insertions(+), 136 deletions(-) delete mode 100755 logger/prepare.sh diff --git a/Makefile b/Makefile index 45f84b615..8e294f116 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,6 @@ GATEWAY_TAG?=$(TAG) LOGGER_TAG?=$(TAG) PROVISIONER_TAG?=$(TAG) RESOURCE_RECORDER_TAG?=$(TAG) -RESOURCE_RECORDER_TAG?=$(TAG) DOCKER_BUILD?=docker buildx build ifeq ($(CI),true) diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 335145ea3..464e90763 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -843,12 +843,9 @@ impl Shuttle { while let Ok(message) = rx.recv() { match message { Message::TextLine(line) => println!("{line}"), - Message::CompilerMessage(message) => { - if let Some(rendered) = message.message.rendered { - println!("{rendered}"); - } + message => { + trace!("skipping cargo line: {message:?}") } - _ => {} } } }); diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 41179dbb9..c7f2d89ab 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -28,4 +28,3 @@ tokio = { version = "1", features = ["full"] } default = [] frameworks = [] next = [] -setup-tracing = [] diff --git a/codegen/src/shuttle_main/mod.rs b/codegen/src/shuttle_main/mod.rs index f400a633c..475c50915 100644 --- a/codegen/src/shuttle_main/mod.rs +++ b/codegen/src/shuttle_main/mod.rs @@ -13,52 +13,9 @@ pub(crate) fn r#impl(_attr: TokenStream, item: TokenStream) -> TokenStream { let loader = Loader::from_item_fn(&mut fn_decl); - let tracing_setup = if cfg!(feature = "setup-tracing") { - Some(quote! { - use shuttle_runtime::colored::{control, Colorize}; - control::set_override(true); // always apply color - - use shuttle_runtime::tracing_subscriber::prelude::*; - let level = if cfg!(debug_assertions) { - "debug,shuttle=trace,h2=info,tower=info,hyper=info" - } else { - "info,shuttle=trace" - }; - shuttle_runtime::tracing_subscriber::registry() - .with(shuttle_runtime::tracing_subscriber::fmt::layer().without_time()) - .with( - // let user override RUST_LOG in local run if they want to - shuttle_runtime::tracing_subscriber::EnvFilter::try_from_default_env() - // otherwise use our default - .or_else(|_| shuttle_runtime::tracing_subscriber::EnvFilter::try_new(level)) - .unwrap() - ) - .init(); - eprintln!( // stderr to not interfere with runtime's --version output on stdout - "{}\n\ - {}\n\ - To disable the subscriber and use your own,\n\ - remove the default features for {}:\n\ - \n\ - {}\n\ - {}", - "=".repeat(63).yellow(), - "Shuttle's default tracing subscriber is initialized!".yellow().bold(), - "shuttle-runtime".italic(), - r#"shuttle-runtime = { version = "...", default-features = false }"# - .white() - .italic(), - "=".repeat(63).yellow() - ); - }) - } else { - None - }; - quote! { #[tokio::main] async fn main() { - #tracing_setup shuttle_runtime::start(loader).await; } diff --git a/common/src/claims.rs b/common/src/claims.rs index aaed28846..818f805e7 100644 --- a/common/src/claims.rs +++ b/common/src/claims.rs @@ -219,7 +219,7 @@ impl Claim { let mut validation = Validation::new(jsonwebtoken::Algorithm::EdDSA); validation.set_issuer(&[ISS]); - trace!(token, "converting token to claim"); + trace!("converting token to claim"); let mut claim: Self = decode(token, &decoding_key, &validation) .map_err(|err| { error!( diff --git a/common/src/deployment.rs b/common/src/deployment.rs index eedbbd0ce..bc30a9389 100644 --- a/common/src/deployment.rs +++ b/common/src/deployment.rs @@ -31,7 +31,7 @@ pub enum Environment { pub const DEPLOYER_END_MSG_STARTUP_ERR: &str = "Service startup encountered an error"; pub const DEPLOYER_END_MSG_BUILD_ERR: &str = "Service build encountered an error"; pub const DEPLOYER_END_MSG_CRASHED: &str = "Service encountered an error and crashed"; -pub const DEPLOYER_END_MSG_STOPPED: &str = "Service was stopped by the user"; +pub const DEPLOYER_END_MSG_STOPPED: &str = "Service was stopped by the user"; // don't include this in end messages so that logs are not stopped too early pub const DEPLOYER_END_MSG_COMPLETED: &str = "Service finished running all on its own"; pub const DEPLOYER_RUNTIME_START_RESPONSE: &str = "Runtime started successully"; @@ -40,11 +40,8 @@ pub const DEPLOYER_END_MESSAGES_BAD: &[&str] = &[ DEPLOYER_END_MSG_BUILD_ERR, DEPLOYER_END_MSG_CRASHED, ]; -pub const DEPLOYER_END_MESSAGES_GOOD: &[&str] = &[ - DEPLOYER_END_MSG_STOPPED, - DEPLOYER_END_MSG_COMPLETED, - DEPLOYER_RUNTIME_START_RESPONSE, -]; +pub const DEPLOYER_END_MESSAGES_GOOD: &[&str] = + &[DEPLOYER_END_MSG_COMPLETED, DEPLOYER_RUNTIME_START_RESPONSE]; #[cfg(test)] mod tests { diff --git a/common/src/log.rs b/common/src/log.rs index 3dceef48a..5c6157cac 100644 --- a/common/src/log.rs +++ b/common/src/log.rs @@ -202,11 +202,7 @@ where let metadata = attrs.metadata(); - let message = format!( - "{} Entering span {}", - metadata.level().colored(), - metadata.name().blue(), - ); + let message = format!("{} {}", metadata.level().colored(), metadata.name().blue()); self.log_recorder.record(LogItem::new( details.deployment_id, diff --git a/deployer/src/deployment/mod.rs b/deployer/src/deployment/mod.rs index 8f8ae8dfb..86357021b 100644 --- a/deployer/src/deployment/mod.rs +++ b/deployer/src/deployment/mod.rs @@ -244,7 +244,7 @@ impl DeploymentManager { self.queue_send.send(queued).await.unwrap(); } - #[instrument(skip(self), fields(deployment_id = %built.id, state = %State::Built))] + #[instrument(name = "Starting deployment", skip(self), fields(deployment_id = %built.id, state = %State::Built))] pub async fn run_push(&self, built: Built) { self.run_send.send(built).await.unwrap(); } diff --git a/deployer/src/deployment/queue.rs b/deployer/src/deployment/queue.rs index 014091213..adfe78ea4 100644 --- a/deployer/src/deployment/queue.rs +++ b/deployer/src/deployment/queue.rs @@ -112,7 +112,7 @@ pub async fn task( } } -#[instrument(skip(_id), fields(deployment_id = %_id, state = %State::Crashed))] +#[instrument(name = "Build failed", skip(_id), fields(deployment_id = %_id, state = %State::Crashed))] fn build_failed(_id: &Uuid, error: impl std::error::Error + 'static) { error!( error = &error as &dyn std::error::Error, @@ -120,9 +120,8 @@ fn build_failed(_id: &Uuid, error: impl std::error::Error + 'static) { ); } -#[instrument(skip(queue_client), fields(state = %State::Queued))] +#[instrument(name = "Waiting for queue slot", skip(queue_client), fields(deployment_id = %id, state = %State::Queued))] async fn wait_for_queue(queue_client: impl BuildQueueClient, id: Uuid) -> Result<()> { - trace!("getting a build slot"); loop { let got_slot = queue_client.get_slot(id).await?; @@ -138,6 +137,7 @@ async fn wait_for_queue(queue_client: impl BuildQueueClient, id: Uuid) -> Result Ok(()) } +#[instrument(name = "Releasing queue slot", skip(queue_client), fields(deployment_id = %id))] async fn remove_from_queue(queue_client: impl BuildQueueClient, id: Uuid) { match queue_client.release_slot(id).await { Ok(_) => {} @@ -148,7 +148,7 @@ async fn remove_from_queue(queue_client: impl BuildQueueClient, id: Uuid) { } } -#[instrument(skip(run_send), fields(deployment_id = %built.id, state = %State::Built))] +#[instrument(name = "Starting deployment", skip(run_send), fields(deployment_id = %built.id, state = %State::Built))] async fn promote_to_run(mut built: Built, run_send: RunSender) { let cx = Span::current().context(); @@ -173,7 +173,7 @@ pub struct Queued { } impl Queued { - #[instrument(skip(self, storage_manager, deployment_updater, log_recorder, secret_recorder), fields(deployment_id = %self.id, state = %State::Building))] + #[instrument(name = "Building project", skip(self, storage_manager, deployment_updater, log_recorder, secret_recorder), fields(deployment_id = %self.id, state = %State::Building))] async fn handle( self, storage_manager: ArtifactsStorageManager, @@ -181,14 +181,11 @@ impl Queued { log_recorder: impl LogRecorder, secret_recorder: impl SecretRecorder, ) -> Result { - info!("Extracting received data"); - let project_path = storage_manager.service_build_path(&self.service_name)?; + info!("Extracting files"); extract_tar_gz_data(self.data.as_slice(), &project_path).await?; - info!("Building deployment"); - let (tx, rx): (crossbeam_channel::Sender, _) = crossbeam_channel::bounded(0); tokio::task::spawn_blocking(move || { @@ -208,6 +205,7 @@ impl Queued { let project_path = project_path.canonicalize()?; + info!("Building deployment"); // Currently returns the first found shuttle service in a given workspace. let built_service = build_deployment(&project_path, tx.clone()).await?; @@ -220,11 +218,7 @@ impl Queued { set_secrets(secrets, &self.service_id, secret_recorder).await?; if self.will_run_tests { - info!( - build_line = "Running tests before starting up", - "Running deployment's unit tests" - ); - + info!("Running tests before starting up"); run_pre_deploy_tests(&project_path, tx).await?; } @@ -359,17 +353,16 @@ async fn run_pre_deploy_tests( let (read, write) = pipe::pipe(); let project_path = project_path.to_owned(); - // This needs to be on a separate thread, else deployer will block (reason currently unknown :D) tokio::task::spawn_blocking(move || { - for message in Message::parse_stream(read) { - match message { - Ok(message) => { - if let Err(error) = tx.send(message) { + for line in read.lines() { + match line { + Ok(line) => { + if let Err(error) = tx.send(Message::TextLine(line)) { error!("failed to send cargo message on channel: {error}"); } } Err(error) => { - error!("failed to parse cargo message: {error}"); + error!("failed to read cargo output line: {error}"); } } } @@ -377,9 +370,12 @@ async fn run_pre_deploy_tests( let mut cmd = Command::new("cargo") .arg("test") + // We set the tests to build with the release profile since deployments compile + // with the release profile by default. This means crates don't need to be + // recompiled in debug mode for the tests, reducing memory usage during deployment. .arg("--release") .arg("--jobs=4") - .arg("--message-format=json") + .arg("--color=always") .current_dir(project_path) .stdout(Stdio::piped()) .spawn() diff --git a/deployer/src/deployment/run.rs b/deployer/src/deployment/run.rs index fa383329f..613f45068 100644 --- a/deployer/src/deployment/run.rs +++ b/deployer/src/deployment/run.rs @@ -30,7 +30,7 @@ use tokio::{ task::{JoinHandle, JoinSet}, }; use tonic::{transport::Channel, Code}; -use tracing::{debug, debug_span, error, info, instrument, trace, warn, Instrument}; +use tracing::{debug, debug_span, error, info, instrument, warn, Instrument}; use tracing_opentelemetry::OpenTelemetrySpanExt; use ulid::Ulid; use uuid::Uuid; @@ -147,7 +147,7 @@ pub async fn task( #[instrument(skip(active_deployment_getter, runtime_manager))] async fn kill_old_deployments( service_id: Ulid, - deployment_id: Uuid, + __deployment_id: Uuid, // prefixed to not catch this span in DeploymentLogLayer active_deployment_getter: impl ActiveDeploymentsGetter, runtime_manager: Arc>, ) -> Result<()> { @@ -159,29 +159,29 @@ async fn kill_old_deployments( .await .map_err(|e| Error::OldCleanup(Box::new(e)))? .into_iter() - .filter(|old_id| old_id != &deployment_id) + .filter(|old_id| old_id != &__deployment_id) { - trace!(%old_id, "stopping old deployment"); + info!("stopping old deployment (id {old_id})"); if !guard.kill(&old_id).await { - warn!(id = %old_id, "failed to kill old deployment"); + warn!("failed to kill old deployment (id {old_id})"); } } Ok(()) } -#[instrument(skip(_id), fields(deployment_id = %_id, state = %State::Completed))] +#[instrument(name = "Cleaning up completed deployment", skip(_id), fields(deployment_id = %_id, state = %State::Completed))] fn completed_cleanup(_id: &Uuid) { info!("{}", DEPLOYER_END_MSG_COMPLETED); } -#[instrument(skip(_id), fields(deployment_id = %_id, state = %State::Stopped))] +#[instrument(name = "Cleaning up stopped deployment", skip(_id), fields(deployment_id = %_id, state = %State::Stopped))] fn stopped_cleanup(_id: &Uuid) { info!("{}", DEPLOYER_END_MSG_STOPPED); } -#[instrument(skip(_id), fields(deployment_id = %_id, state = %State::Crashed))] +#[instrument(name = "Cleaning up crashed deployment", skip(_id), fields(deployment_id = %_id, state = %State::Crashed))] fn crashed_cleanup(_id: &Uuid, error: impl std::error::Error + 'static) { error!( error = &error as &dyn std::error::Error, @@ -189,7 +189,7 @@ fn crashed_cleanup(_id: &Uuid, error: impl std::error::Error + 'static) { ); } -#[instrument(skip(_id), fields(deployment_id = %_id, state = %State::Crashed))] +#[instrument(name = "Cleaning up startup crashed deployment", skip(_id), fields(deployment_id = %_id, state = %State::Crashed))] fn start_crashed_cleanup(_id: &Uuid, error: impl std::error::Error + 'static) { error!( error = &error as &dyn std::error::Error, @@ -219,7 +219,7 @@ pub struct Built { } impl Built { - #[instrument(skip(self, storage_manager, secret_getter, resource_manager, runtime_manager, deployment_updater, kill_old_deployments, cleanup), fields(deployment_id = %self.id, state = %State::Loading))] + #[instrument(name = "Loading resources", skip(self, storage_manager, secret_getter, resource_manager, runtime_manager, deployment_updater, kill_old_deployments, cleanup), fields(deployment_id = %self.id, state = %State::Loading))] #[allow(clippy::too_many_arguments)] async fn handle( self, @@ -304,14 +304,7 @@ async fn load( mut runtime_client: RuntimeClient>>, claim: Claim, ) -> Result<()> { - info!( - "loading project from: {}", - executable_path - .clone() - .into_os_string() - .into_string() - .unwrap_or_default() - ); + info!("Loading resources"); let resources = resource_manager .get_resources(&service_id, claim.clone()) @@ -398,7 +391,7 @@ async fn load( } // TODO: add ticket to add deployment_id to more functions that need to be instrumented in deployer. -#[instrument(skip(runtime_client, deployment_updater, cleanup), fields(deployment_id = %id, state = %State::Running))] +#[instrument(name = "Starting service", skip(runtime_client, deployment_updater, cleanup), fields(deployment_id = %id, state = %State::Running))] async fn run( id: Uuid, service_name: String, @@ -423,13 +416,12 @@ async fn run( .unwrap() .into_inner(); - info!("starting service"); let response = runtime_client.start(start_request).await; match response { Ok(response) => { if response.into_inner().success { - info!(DEPLOYER_RUNTIME_START_RESPONSE); + info!("{}", DEPLOYER_RUNTIME_START_RESPONSE); } // Wait for stop reason diff --git a/deployer/src/handlers/mod.rs b/deployer/src/handlers/mod.rs index 5b572a440..77aef5b50 100644 --- a/deployer/src/handlers/mod.rs +++ b/deployer/src/handlers/mod.rs @@ -10,11 +10,11 @@ use axum::middleware::{self, from_extractor}; use axum::routing::{get, post, Router}; use axum::Json; use bytes::Bytes; -use chrono::Utc; +use chrono::{SecondsFormat, Utc}; use fqdn::FQDN; use hyper::{Request, StatusCode, Uri}; use serde::{de::DeserializeOwned, Deserialize}; -use tracing::{error, field, instrument, trace, warn}; +use tracing::{error, field, info, info_span, instrument, trace, warn}; use ulid::Ulid; use utoipa::{IntoParams, OpenApi}; use utoipa_swagger_ui::SwaggerUi; @@ -117,7 +117,6 @@ impl RouterBuilder { get(get_service.layer(ScopedLayer::new(vec![Scope::Service]))) .post( create_service - .layer(Extension(project_id)) .layer(DefaultBodyLimit::max(CREATE_SERVICE_BODY_LIMIT)) .layer(ScopedLayer::new(vec![Scope::ServiceCreate])), ) @@ -343,12 +342,31 @@ pub async fn create_service( Path((project_name, service_name)): Path<(String, String)>, Rmp(deployment_req): Rmp, ) -> Result> { + let id = Uuid::new_v4(); + let now = Utc::now(); + + let span = info_span!( + "Starting deployment", + deployment_id = %id, + ); + let service = persistence.get_or_create_service(&service_name).await?; + let pid = persistence.project_id(); + + span.in_scope(|| { + info!("Deployment ID: {}", id); + info!("Service ID: {}", service.id); + info!("Service name: {}", service.name); + info!("Project ID: {}", pid); + info!("Project name: {}", project_name); + info!("Date: {}", now.to_rfc3339_opts(SecondsFormat::Secs, true)); + }); + let deployment = Deployment { - id: Uuid::new_v4(), + id, service_id: service.id, state: State::Queued, - last_update: Utc::now(), + last_update: now, address: None, is_next: false, git_commit_id: deployment_req @@ -368,7 +386,7 @@ pub async fn create_service( id: deployment.id, service_name: service.name, service_id: deployment.service_id, - project_id: persistence.project_id(), + project_id: pid, data: deployment_req.data, will_run_tests: !deployment_req.no_test, tracing_context: Default::default(), @@ -732,7 +750,7 @@ pub async fn clean_project( .service_build_path(&project_name) .map_err(anyhow::Error::new)?; - let lines = clean_crate(&project_path, true).await?; + let lines = clean_crate(&project_path).await?; Ok(Json(lines)) } diff --git a/logger/prepare.sh b/logger/prepare.sh deleted file mode 100755 index 6a52d3030..000000000 --- a/logger/prepare.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh - -############################################################################### -# This file is used by our common Containerfile incase the container for this # -# service might need some extra preparation steps for its final image # -############################################################################### - -# Nothing to prepare in container image here diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 5dffd18db..559da8de3 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -154,10 +154,10 @@ pub mod runtime { args }; - trace!( - "Spawning runtime process {:?} {:?}", - runtime_executable_path, - args + info!( + "Spawning runtime process: {} {}", + runtime_executable_path.display(), + args.join(" ") ); let runtime = process::Command::new(runtime_executable_path) .args(&args) diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index af60fb045..9b8a79f65 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -73,5 +73,4 @@ setup-tracing = [ "tracing-subscriber/default", "tracing-subscriber/env-filter", "colored", - "shuttle-codegen/setup-tracing", ] diff --git a/runtime/src/alpha/mod.rs b/runtime/src/alpha/mod.rs index 2a552f96a..87027941b 100644 --- a/runtime/src/alpha/mod.rs +++ b/runtime/src/alpha/mod.rs @@ -64,6 +64,43 @@ pub async fn start(loader: impl Loader + Send + 'static) { } }; + // this is handled after arg parsing to not interfere with --version above + #[cfg(feature = "setup-tracing")] + { + use colored::Colorize; + use tracing_subscriber::prelude::*; + + colored::control::set_override(true); // always apply color + + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().without_time()) + .with( + // let user override RUST_LOG in local run if they want to + tracing_subscriber::EnvFilter::try_from_default_env() + // otherwise use our default + .or_else(|_| tracing_subscriber::EnvFilter::try_new("info,shuttle=trace")) + .unwrap(), + ) + .init(); + + println!( + "{}\n\ + {}\n\ + To disable the subscriber and use your own,\n\ + turn off the default features for {}:\n\ + \n\ + {}\n\ + {}", + "=".repeat(63).yellow(), + "Shuttle's default tracing subscriber is initialized!" + .yellow() + .bold(), + "shuttle-runtime".italic(), + r#"shuttle-runtime = { version = "...", default-features = false }"#.italic(), + "=".repeat(63).yellow(), + ); + } + let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), args.port); let provisioner_address = args.provisioner_address; diff --git a/service/src/builder.rs b/service/src/builder.rs index ed7f950b9..c459cdc46 100644 --- a/service/src/builder.rs +++ b/service/src/builder.rs @@ -141,19 +141,16 @@ pub async fn build_workspace( Ok(runtimes) } -pub async fn clean_crate(project_path: &Path, release_mode: bool) -> anyhow::Result> { +pub async fn clean_crate(project_path: &Path) -> anyhow::Result> { let project_path = project_path.to_owned(); let manifest_path = project_path.join("Cargo.toml"); if !manifest_path.exists() { bail!("failed to read the Shuttle project manifest"); } - let profile = if release_mode { "release" } else { "dev" }; let output = tokio::process::Command::new("cargo") .arg("clean") .arg("--manifest-path") .arg(manifest_path.to_str().unwrap()) - .arg("--profile") - .arg(profile) .output() .await .unwrap(); @@ -225,13 +222,11 @@ async fn compile( let target_path = target_path.into(); let mut cargo = tokio::process::Command::new("cargo"); - - let (reader, writer) = os_pipe::pipe()?; - let writer_clone = writer.try_clone()?; - cargo.stdout(writer); - cargo.stderr(writer_clone); - - cargo.arg("build").arg("--manifest-path").arg(manifest_path); + cargo + .arg("build") + .arg("--manifest-path") + .arg(manifest_path) + .arg("--color=always"); // piping disables auto color if deployment { cargo.arg("-j").arg(4.to_string()); @@ -253,6 +248,11 @@ async fn compile( cargo.arg("--target").arg("wasm32-wasi"); } + let (reader, writer) = os_pipe::pipe()?; + let writer_clone = writer.try_clone()?; + cargo.stdout(writer); + cargo.stderr(writer_clone); + let mut handle = cargo.spawn()?; tokio::task::spawn_blocking(move || {