diff --git a/crates/stages/stages/src/stages/s3/downloader/fetch.rs b/crates/stages/stages/src/stages/s3/downloader/fetch.rs
index ae329f02e1ce..1a31c8bc0b51 100644
--- a/crates/stages/stages/src/stages/s3/downloader/fetch.rs
+++ b/crates/stages/stages/src/stages/s3/downloader/fetch.rs
@@ -1,4 +1,4 @@
-use crate::stages::s3::downloader::worker::spawn_workers;
+use crate::stages::s3::downloader::{worker::spawn_workers, RemainingChunkRange};
use super::{
error::DownloaderError,
@@ -101,9 +101,9 @@ pub async fn fetch(
let worker = workers.get(&available_worker).expect("should exist");
match missing_chunks.next() {
- Some((chunk_index, (start, end))) => {
+ Some(RemainingChunkRange { index, start, end }) => {
debug!(target: "sync::stages::s3::downloader", ?available_worker, start, end, "Worker download request.");
- let _ = worker.send(WorkerRequest::Download { chunk_index, start, end });
+ let _ = worker.send(WorkerRequest::Download { chunk_index: index, start, end });
}
None => {
let _ = worker.send(WorkerRequest::Finish);
diff --git a/crates/stages/stages/src/stages/s3/downloader/meta.rs b/crates/stages/stages/src/stages/s3/downloader/meta.rs
index 290286e4c334..7ff4213fffc3 100644
--- a/crates/stages/stages/src/stages/s3/downloader/meta.rs
+++ b/crates/stages/stages/src/stages/s3/downloader/meta.rs
@@ -1,13 +1,14 @@
-use super::error::DownloaderError;
+use super::{error::DownloaderError, RemainingChunkRange};
use serde::{Deserialize, Serialize};
use std::{
fs::File,
+ ops::RangeInclusive,
path::{Path, PathBuf},
};
use tracing::info;
/// Tracks download progress and manages chunked downloads for resumable file transfers.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug)]
pub struct Metadata {
/// Total file size
pub total_size: usize,
@@ -16,11 +17,10 @@ pub struct Metadata {
/// Download chunk size. Default 150MB.
pub chunk_size: usize,
/// Remaining download ranges for each chunk.
- /// - `Some((start, end))`: Start and end indices to be downloaded.
+ /// - `Some(RangeInclusive)`: range to be downloaded.
/// - `None`: Chunk fully downloaded.
- chunks: Vec