Skip to content

Commit

Permalink
Remove tqdm
Browse files Browse the repository at this point in the history
  • Loading branch information
antares-sw committed Jan 10, 2025
1 parent 4a65ba4 commit 02a8de5
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 57 deletions.
1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,4 @@ dependencies = [
"docker>=7.1.0",
"google-cloud-storage>=2.19.0",
"toml>=0.10.2",
"tqdm>=4.67.1",
]
47 changes: 14 additions & 33 deletions storage.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import io
import os
import subprocess
from google.cloud.storage import Client, Bucket
from tqdm import tqdm

def create_tar(paths: list[str], tar_name: str) -> None:
"""Create an uncompressed tarball without including the full path of the files inside."""
Expand All @@ -28,41 +26,24 @@ def extract_tar(tar_path: str, extract_to_path: str) -> None:
print(f"Extraction complete: {extract_to_path}")


def download_from_bucket(storage_client: Client, bucket_name: str, source_blob_name: str, destination_file_name: str) -> None:
"""Download a file from Google Cloud Storage with progress tracking."""
try:
bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(source_blob_name)

with open(destination_file_name, 'wb') as f:
with tqdm.wrapattr(f, "write", total=blob.size, desc=f"Downloading {source_blob_name}") as file_obj:
blob.download_to_file(file_obj)

print(f"File {source_blob_name} downloaded to {destination_file_name}.")
except Exception as e:
print(f"Error downloading file: {e}")
def download_from_bucket(storage_client: Client, bucket_name: str, bucket_path: str, local_path: str) -> None:
"""Download the snapshot tar file from Google Cloud Storage."""
bucket: Bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(bucket_path)
print(f"Downloading {bucket_path} from gs://{bucket_name}...")

# Download the tar file to the local path
blob.download_to_filename(local_path)
print(f"Downloaded {bucket_path} to {local_path}")


def upload_file_to_bucket(
storage_client: Client,
bucket_name: str,
source_file: str,
destination_blob_name: str
) -> None:
"""Upload a file to the specified Google Cloud Storage bucket with progress tracking."""
def upload_file_to_bucket(storage_client: Client, bucket_name: str, source_file: str, destination_blob_name: str) -> None:
"""Upload a file to the specified Google Cloud Storage bucket."""
try:
bucket = storage_client.get_bucket(bucket_name)
print(f"Start uploading to {destination_blob_name}.")
bucket: Bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(destination_blob_name)

file_size = os.path.getsize(source_file)

with tqdm(total=file_size, unit='B', unit_scale=True, desc=f"Uploading {source_file}") as pbar:
with open(source_file, "rb") as f:
chunk_size = 1024 * 1024 * 32
for chunk in iter(lambda: f.read(chunk_size), b""):
blob.upload_from_file(io.BytesIO(chunk), content_type="application/octet-stream")
pbar.update(len(chunk))

blob.upload_from_filename(source_file)
print(f"File {source_file} uploaded to {destination_blob_name}.")
except Exception as e:
print(f"Error uploading file: {e}")
23 changes: 0 additions & 23 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 02a8de5

Please sign in to comment.