fetch: add retry support for downloads
Signed-off-by: NotAShelf <raf@notashelf.dev> Change-Id: I5920652b1f84cd8d03e3f8c9d17e5aa76a6a6964
This commit is contained in:
parent
c0c9d741c1
commit
0048a1cd73
2 changed files with 44 additions and 10 deletions
|
|
@ -38,7 +38,9 @@ pub async fn execute(
|
||||||
let _guard = OperationGuard::new(coordinator, operation_id);
|
let _guard = OperationGuard::new(coordinator, operation_id);
|
||||||
|
|
||||||
// Create fetcher with shelve option
|
// Create fetcher with shelve option
|
||||||
let fetcher = Fetcher::new(".").with_shelve(args.shelve);
|
let fetcher = Fetcher::new(".")
|
||||||
|
.with_shelve(args.shelve)
|
||||||
|
.with_retry(args.retry);
|
||||||
|
|
||||||
// Fetch all projects (progress indicators handled in fetch.rs)
|
// Fetch all projects (progress indicators handled in fetch.rs)
|
||||||
fetcher.fetch_all(&lockfile, &config).await?;
|
fetcher.fetch_all(&lockfile, &config).await?;
|
||||||
|
|
|
||||||
50
src/fetch.rs
50
src/fetch.rs
|
|
@ -19,17 +19,19 @@ use crate::{
|
||||||
const MAX_CONCURRENT_DOWNLOADS: usize = 8;
|
const MAX_CONCURRENT_DOWNLOADS: usize = 8;
|
||||||
|
|
||||||
pub struct Fetcher {
|
pub struct Fetcher {
|
||||||
client: Client,
|
client: Client,
|
||||||
base_path: PathBuf,
|
base_path: PathBuf,
|
||||||
shelve: bool,
|
shelve: bool,
|
||||||
|
retry_count: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Fetcher {
|
impl Fetcher {
|
||||||
pub fn new<P: AsRef<Path>>(base_path: P) -> Self {
|
pub fn new<P: AsRef<Path>>(base_path: P) -> Self {
|
||||||
Self {
|
Self {
|
||||||
client: Client::new(),
|
client: Client::new(),
|
||||||
base_path: base_path.as_ref().to_path_buf(),
|
base_path: base_path.as_ref().to_path_buf(),
|
||||||
shelve: false,
|
shelve: false,
|
||||||
|
retry_count: 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -38,6 +40,11 @@ impl Fetcher {
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub const fn with_retry(mut self, retry_count: u32) -> Self {
|
||||||
|
self.retry_count = retry_count;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn sync(&self, lockfile: &LockFile, config: &Config) -> Result<()> {
|
pub async fn sync(&self, lockfile: &LockFile, config: &Config) -> Result<()> {
|
||||||
self.fetch_all(lockfile, config).await
|
self.fetch_all(lockfile, config).await
|
||||||
}
|
}
|
||||||
|
|
@ -96,6 +103,7 @@ impl Fetcher {
|
||||||
client,
|
client,
|
||||||
base_path,
|
base_path,
|
||||||
shelve: false, // Shelving happens at sync level, not per-project
|
shelve: false, // Shelving happens at sync level, not per-project
|
||||||
|
retry_count: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
let result = fetcher.fetch_project(&project, lockfile, config).await;
|
let result = fetcher.fetch_project(&project, lockfile, config).await;
|
||||||
|
|
@ -389,14 +397,39 @@ impl Fetcher {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Download a file from URL to target path
|
/// Download a file from URL to target path with retry
|
||||||
async fn download_file(&self, url: &str, target_path: &Path) -> Result<()> {
|
async fn download_file(&self, url: &str, target_path: &Path) -> Result<()> {
|
||||||
// Create parent directory
|
// Create parent directory
|
||||||
if let Some(parent) = target_path.parent() {
|
if let Some(parent) = target_path.parent() {
|
||||||
fs::create_dir_all(parent)?;
|
fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download file
|
let max_attempts = self.retry_count.saturating_add(1);
|
||||||
|
|
||||||
|
for attempt in 0..max_attempts {
|
||||||
|
match self.download_single_attempt(url, target_path).await {
|
||||||
|
Ok(()) => return Ok(()),
|
||||||
|
Err(e) if attempt < self.retry_count => {
|
||||||
|
log::warn!(
|
||||||
|
"Download attempt {}/{} failed for {}, retrying...",
|
||||||
|
attempt + 1,
|
||||||
|
max_attempts,
|
||||||
|
url
|
||||||
|
);
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
|
||||||
|
},
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn download_single_attempt(
|
||||||
|
&self,
|
||||||
|
url: &str,
|
||||||
|
target_path: &Path,
|
||||||
|
) -> Result<()> {
|
||||||
let response = self.client.get(url).send().await?;
|
let response = self.client.get(url).send().await?;
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
|
|
@ -405,7 +438,6 @@ impl Fetcher {
|
||||||
|
|
||||||
let bytes = response.bytes().await?;
|
let bytes = response.bytes().await?;
|
||||||
|
|
||||||
// Write to temporary file first (atomic write)
|
|
||||||
let temp_path = target_path.with_extension("tmp");
|
let temp_path = target_path.with_extension("tmp");
|
||||||
fs::write(&temp_path, bytes)?;
|
fs::write(&temp_path, bytes)?;
|
||||||
fs::rename(temp_path, target_path)?;
|
fs::rename(temp_path, target_path)?;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue