NotAShelf/circus
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fc-server: return INSUFFICIENT_STORAGE for disk space errors

Browse source 
Eugh. Need I explain this? We now return a HTTP 507 INSUFFICIENT_STORAGE
for disk space errors rather than letting it all go to shit. Some better
error handling for disk/IO errors. See previous commit also.

Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: Ieb6d6415ff8a7209590217933c6992796a6a6964
This commit is contained in:
raf 2026-02-05 22:41:00 +03:00
commit b3125a89d8
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
1 changed files with 103 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

122
crates/server/src/error.rs
View file

@ -33,10 +33,45 @@ impl IntoResponse for ApiError {
(StatusCode::FORBIDDEN, "FORBIDDEN", msg.clone())
},
CiError::NixEval(msg) => {
let is_disk_full =
msg.to_lowercase().contains("no space left on device")
|| msg.to_lowercase().contains("disk full")
|| msg.to_lowercase().contains("enospc")
|| msg.to_lowercase().contains("cannot create directory")
|| msg.to_lowercase().contains("sqlite");
if is_disk_full {
(
StatusCode::INSUFFICIENT_STORAGE,
"DISK_FULL",
format!(
"{}\n\nDISK SPACE ISSUE DETECTED:\nThe server has run out of \
disk space. Please free up space:\n- Run `nix-collect-garbage \
-d` to clean the Nix store\n- Clear the evaluator work \
directory: `rm -rf /tmp/fc-evaluator/*`\n- Clear build logs if \
configured",
msg
),
)
} else {
(
StatusCode::UNPROCESSABLE_ENTITY,
"NIX_EVAL_ERROR",
msg.clone(),
)
}
},
CiError::DiskSpace(msg) => {
(
StatusCode::UNPROCESSABLE_ENTITY,
"NIX_EVAL_ERROR",
msg.clone(),
StatusCode::INSUFFICIENT_STORAGE,
"DISK_FULL",
format!(
"{}\n\nDISK SPACE ISSUE:\nThe server is running low on disk \
space. Please free up space:\n- Run `nix-collect-garbage -d` to \
clean the Nix store\n- Clear the evaluator work directory\n- \
Clear build logs if configured",
msg
),
)
},
CiError::Build(msg) => {
@ -51,11 +86,24 @@ impl IntoResponse for ApiError {
},
CiError::Database(e) => {
tracing::error!(error = %e, "Database error in API handler");
(
StatusCode::INTERNAL_SERVER_ERROR,
"DATABASE_ERROR",
"Internal database error".to_string(),
)
if e.to_string().contains("disk") || e.to_string().contains("space") {
(
StatusCode::INSUFFICIENT_STORAGE,
"DISK_FULL",
format!(
"Database error: {}\n\nDISK SPACE ISSUE:\nThe server is running \
low on disk space.",
e
),
)
} else {
(
StatusCode::INTERNAL_SERVER_ERROR,
"DATABASE_ERROR",
"Internal database error".to_string(),
)
}
},
CiError::Git(e) => {
tracing::error!(error = %e, "Git error in API handler");
@ -75,23 +123,59 @@ impl IntoResponse for ApiError {
},
CiError::Io(e) => {
tracing::error!(error = %e, "IO error in API handler");
(
StatusCode::INTERNAL_SERVER_ERROR,
"IO_ERROR",
format!("IO error: {e}"),
)
let msg = e.to_string();
if msg.contains("No space left on device")
|| msg.contains("disk full")
|| msg.contains("ENOSPC")
{
(
StatusCode::INSUFFICIENT_STORAGE,
"DISK_FULL",
format!(
"IO error: {}\n\nDISK SPACE ISSUE DETECTED:\nThe server has run \
out of disk space. Please free up space:\n- Run \
`nix-collect-garbage -d` to clean the Nix store\n- Clear the \
evaluator work directory: `rm -rf /tmp/fc-evaluator/*`\n- \
Clear build logs if configured",
msg
),
)
} else {
(
StatusCode::INTERNAL_SERVER_ERROR,
"IO_ERROR",
format!("IO error: {e}"),
)
}
},
CiError::Internal(msg) => {
tracing::error!(message = %msg, "Internal error in API handler");
(
StatusCode::INTERNAL_SERVER_ERROR,
"INTERNAL_ERROR",
msg.clone(),
)
if msg.to_lowercase().contains("disk")
|| msg.to_lowercase().contains("space")
|| msg.to_lowercase().contains("storage")
{
(
StatusCode::INSUFFICIENT_STORAGE,
"DISK_FULL",
format!(
"{}\n\nDISK SPACE ISSUE:\nThe server is running low on disk \
space. Please free up space.",
msg
),
)
} else {
(
StatusCode::INTERNAL_SERVER_ERROR,
"INTERNAL_ERROR",
msg.clone(),
)
}
},
};
if status.is_server_error() {
if status.is_server_error() || status == StatusCode::INSUFFICIENT_STORAGE {
tracing::warn!(
status = %status,
code = code,