Skip to content

Commit

Permalink
fix: check for draining before installing/creating dns (#773)
Browse files Browse the repository at this point in the history
<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->
  • Loading branch information
MasterPtato committed May 15, 2024
1 parent 163beaf commit cbe450b
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 46 deletions.
32 changes: 17 additions & 15 deletions svc/pkg/cluster/worker/src/workers/server_dns_create.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use crate::util::CloudflareError;
struct Server {
datacenter_id: Uuid,
public_ip: IpAddr,
cloud_destroy_ts: Option<i64>,
is_destroying_or_draining: bool,
}

#[worker(name = "cluster-server-dns-create")]
Expand Down Expand Up @@ -44,18 +44,6 @@ async fn inner(
) -> GlobalResult<()> {
let server_id = unwrap_ref!(ctx.server_id).as_uuid();

let server = sql_fetch_one!(
[ctx, Server, @tx tx]
"
SELECT
datacenter_id, public_ip, cloud_destroy_ts
FROM db_cluster.servers
WHERE server_id = $1
",
server_id,
)
.await?;

// Lock row
sql_execute!(
[ctx, @tx tx]
Expand All @@ -70,8 +58,22 @@ async fn inner(
)
.await?;

if server.cloud_destroy_ts.is_some() {
tracing::info!("server marked for deletion, not creating dns record");
let server = sql_fetch_one!(
[ctx, Server, @tx tx]
"
SELECT
datacenter_id,
public_ip,
(cloud_destroy_ts IS NOT NULL OR drain_ts IS NOT NULL) AS is_destroying_or_draining
FROM db_cluster.servers
WHERE server_id = $1
",
server_id,
)
.await?;

if server.is_destroying_or_draining {
tracing::info!("server marked for deletion/drain, not creating dns record");
return Ok(());
}

Expand Down
11 changes: 6 additions & 5 deletions svc/pkg/cluster/worker/src/workers/server_dns_delete.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use proto::backend::pkg::*;
use crate::util::CloudflareError;

#[derive(sqlx::FromRow)]
struct DnsRecord {
struct DnsRecordRow {
dns_record_id: Option<String>,
secondary_dns_record_id: Option<String>,
}
Expand Down Expand Up @@ -44,7 +44,7 @@ async fn inner(
let server_id = unwrap_ref!(ctx.server_id).as_uuid();

let row = sql_fetch_optional!(
[ctx, DnsRecord, @tx tx]
[ctx, DnsRecordRow, @tx tx]
"
SELECT dns_record_id, secondary_dns_record_id
FROM db_cluster.servers_cloudflare
Expand All @@ -57,7 +57,8 @@ async fn inner(
util::timestamp::now(),
)
.await?;
let Some(DnsRecord {

let Some(DnsRecordRow {
dns_record_id,
secondary_dns_record_id,
}) = row
Expand Down Expand Up @@ -87,7 +88,7 @@ async fn inner(
tracing::warn!(%zone_id, %record_id, "dns record not found");
} else {
res?;
tracing::warn!(%record_id, "deleted dns record");
tracing::info!(%record_id, "deleted dns record");
}
} else {
tracing::warn!("server has no primary dns record");
Expand All @@ -102,7 +103,7 @@ async fn inner(
})
.await?;

tracing::warn!(%record_id, "deleted secondary dns record");
tracing::info!(%record_id, "deleted secondary dns record");
} else {
tracing::warn!("server has no secondary dns record");
}
Expand Down
30 changes: 4 additions & 26 deletions svc/pkg/cluster/worker/src/workers/server_install/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,22 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
}

if let Some(server_id) = ctx.server_id {
let (is_destroying,) = sql_fetch_one!(
let (is_destroying_or_draining,) = sql_fetch_one!(
[ctx, (bool,)]
"
SELECT EXISTS(
SELECT 1
FROM db_cluster.servers
WHERE server_id = $1 AND
cloud_destroy_ts IS NOT NULL
(cloud_destroy_ts IS NOT NULL OR drain_ts IS NOT NULL)
)
",
server_id.as_uuid(),
)
.await?;

if is_destroying {
tracing::info!("server marked for deletion, not installing");
if is_destroying_or_draining {
tracing::info!("server marked for deletion/drain, not installing");
return Ok(());
}
}
Expand Down Expand Up @@ -133,28 +133,6 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
})
.await??;

// Check if destroyed again after installing
if let Some(server_id) = ctx.server_id {
let (is_destroying,) = sql_fetch_one!(
[ctx, (bool,)]
"
SELECT EXISTS(
SELECT 1
FROM db_cluster.servers
WHERE server_id = $1 AND
cloud_destroy_ts IS NOT NULL
)
",
server_id.as_uuid(),
)
.await?;

if is_destroying {
tracing::info!("server marked for deletion, not marking as installed");
return Ok(());
}
}

let request_id = unwrap_ref!(ctx.request_id).as_uuid();
msg!([ctx] cluster::msg::server_install_complete(request_id) {
request_id: ctx.request_id,
Expand Down

0 comments on commit cbe450b

Please sign in to comment.