Skip to content

Commit

Permalink
[cli] ingest all queue fix (#20)
Browse files Browse the repository at this point in the history
* fix ingest-all processing on gz, add test

* patch order of queue check

---------

Co-authored-by: Raffa Tempo <raffa_tempo@github.com>
Co-authored-by: Lucietta MacUrchin <lucietta_macurchin@github.com>
Co-authored-by: Caty Cony <caty_cony@github.com>
4 people authored Jan 30, 2025
1 parent b42086f commit fada7c4
Showing 2 changed files with 11 additions and 12 deletions.
20 changes: 10 additions & 10 deletions src/load.rs
Original file line number Diff line number Diff line change
@@ -44,29 +44,29 @@ pub async fn ingest_all(

// This manifest may be for a .gz file, we should handle here as well
for (_p, m) in archive_map.0.iter() {
info!("checking if we need to decompress");
let (new_unzip_path, temp) = unzip_temp::maybe_handle_gz(&m.archive_dir)?;
let mut better_man = ManifestInfo::new(&new_unzip_path);
better_man.set_info()?;

println!(
"\nProcessing: {:?} with archive: {}",
better_man.contents,
better_man.archive_dir.display()
m.contents,
m.archive_dir.display()
);

let complete = queue::are_all_completed(pool, &better_man.archive_id).await?;
let complete = queue::are_all_completed(pool, &m.archive_id).await?;

if !complete {
info!("checking if we need to decompress");
let (new_unzip_path, temp) = unzip_temp::maybe_handle_gz(&m.archive_dir)?;
let mut better_man = ManifestInfo::new(&new_unzip_path);
better_man.set_info()?;

let batch_tx_return = try_load_one_archive(&better_man, pool, batch_size).await?;
println!("SUCCESS: {}", batch_tx_return);
drop(temp);
} else {
info!(
"archive complete (or not in queue): {}",
better_man.archive_dir.display()
m.archive_dir.display()
);
}
drop(temp);
}

Ok(())
3 changes: 1 addition & 2 deletions src/unzip_temp.rs
Original file line number Diff line number Diff line change
@@ -125,9 +125,8 @@ pub fn maybe_handle_gz(archive_path: &Path) -> Result<(PathBuf, Option<TempPath>
// maybe stuff isn't unzipped yet
let pattern = format!("{}/*.*.gz", archive_path.display());
if glob(&pattern)?.count() > 0 {
let mut temp_dir = TempPath::new();
let temp_dir = TempPath::new();
temp_dir.create_as_dir()?;
temp_dir.persist();

// need to preserve the parent dir name in temp, since the manifest files reference it.
let dir_name = archive_path.file_name().unwrap().to_str().unwrap();

0 comments on commit fada7c4

Please sign in to comment.