fc-queue-runner: integrate GC pinning and machine health tracking
`gc_loop` queries pinned build IDs before cleanup. `try_remote_build` calls `record_success`/`record_failure` per builder. Signed-off-by: NotAShelf <raf@notashelf.dev> Change-Id: Ia8e20ead3c83b78d787dba518c382f9a6a6a6964
This commit is contained in:
parent
5410fdc044
commit
015360ffcf
2 changed files with 26 additions and 4 deletions
|
|
@ -87,7 +87,7 @@ async fn main() -> anyhow::Result<()> {
|
||||||
tracing::error!("Runner loop failed: {e}");
|
tracing::error!("Runner loop failed: {e}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
() = gc_loop(gc_config_for_loop) => {}
|
() = gc_loop(gc_config_for_loop, db.pool().clone()) => {}
|
||||||
() = failed_paths_cleanup_loop(db.pool().clone(), failed_paths_ttl, failed_paths_cache) => {}
|
() = failed_paths_cleanup_loop(db.pool().clone(), failed_paths_ttl, failed_paths_cache) => {}
|
||||||
() = cancel_checker_loop(db.pool().clone(), active_builds) => {}
|
() = cancel_checker_loop(db.pool().clone(), active_builds) => {}
|
||||||
() = shutdown_signal() => {
|
() = shutdown_signal() => {
|
||||||
|
|
@ -118,7 +118,7 @@ async fn cleanup_stale_logs(log_dir: &std::path::Path) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn gc_loop(gc_config: GcConfig) {
|
async fn gc_loop(gc_config: GcConfig, pool: sqlx::PgPool) {
|
||||||
if !gc_config.enabled {
|
if !gc_config.enabled {
|
||||||
return std::future::pending().await;
|
return std::future::pending().await;
|
||||||
}
|
}
|
||||||
|
|
@ -127,7 +127,17 @@ async fn gc_loop(gc_config: GcConfig) {
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
tokio::time::sleep(interval).await;
|
tokio::time::sleep(interval).await;
|
||||||
match gc_roots::cleanup_old_roots(&gc_config.gc_roots_dir, max_age) {
|
|
||||||
|
let pinned = match repo::builds::list_pinned_ids(&pool).await {
|
||||||
|
Ok(ids) => ids,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("Failed to fetch pinned build IDs for GC: {e}");
|
||||||
|
std::collections::HashSet::new()
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
match gc_roots::cleanup_old_roots(&gc_config.gc_roots_dir, max_age, &pinned)
|
||||||
|
{
|
||||||
Ok(count) if count > 0 => {
|
Ok(count) if count > 0 => {
|
||||||
tracing::info!(count, "Cleaned up old GC roots");
|
tracing::info!(count, "Cleaned up old GC roots");
|
||||||
// Optionally run nix-collect-garbage
|
// Optionally run nix-collect-garbage
|
||||||
|
|
|
||||||
|
|
@ -427,13 +427,25 @@ async fn try_remote_build(
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(r) => return Some(r),
|
Ok(r) => {
|
||||||
|
if let Err(e) =
|
||||||
|
repo::remote_builders::record_success(pool, builder.id).await
|
||||||
|
{
|
||||||
|
tracing::warn!(builder = %builder.name, "Failed to record builder success: {e}");
|
||||||
|
}
|
||||||
|
return Some(r);
|
||||||
|
},
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
build_id = %build.id,
|
build_id = %build.id,
|
||||||
builder = %builder.name,
|
builder = %builder.name,
|
||||||
"Remote build failed: {e}, trying next builder"
|
"Remote build failed: {e}, trying next builder"
|
||||||
);
|
);
|
||||||
|
if let Err(e) =
|
||||||
|
repo::remote_builders::record_failure(pool, builder.id).await
|
||||||
|
{
|
||||||
|
tracing::warn!(builder = %builder.name, "Failed to record builder failure: {e}");
|
||||||
|
}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue