From 02a8de51420ff0beffef898e3301b693503ea323 Mon Sep 17 00:00:00 2001
From: antares-sw <23400824+antares-sw@users.noreply.github.com>
Date: Fri, 10 Jan 2025 14:41:03 +0300
Subject: [PATCH] Remove tqdm
---
pyproject.toml | 1 -
storage.py | 47 ++++++++++++++---------------------------------
uv.lock | 23 -----------------------
3 files changed, 14 insertions(+), 57 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 92153fa..f91e2d7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,5 +8,4 @@ dependencies = [
"docker>=7.1.0",
"google-cloud-storage>=2.19.0",
"toml>=0.10.2",
- "tqdm>=4.67.1",
]
diff --git a/storage.py b/storage.py
index b6b9123..525efbf 100644
--- a/storage.py
+++ b/storage.py
@@ -1,8 +1,6 @@
-import io
import os
import subprocess
from google.cloud.storage import Client, Bucket
-from tqdm import tqdm
def create_tar(paths: list[str], tar_name: str) -> None:
"""Create an uncompressed tarball without including the full path of the files inside."""
@@ -28,41 +26,24 @@ def extract_tar(tar_path: str, extract_to_path: str) -> None:
print(f"Extraction complete: {extract_to_path}")
-def download_from_bucket(storage_client: Client, bucket_name: str, source_blob_name: str, destination_file_name: str) -> None:
- """Download a file from Google Cloud Storage with progress tracking."""
- try:
- bucket = storage_client.get_bucket(bucket_name)
- blob = bucket.blob(source_blob_name)
-
- with open(destination_file_name, 'wb') as f:
- with tqdm.wrapattr(f, "write", total=blob.size, desc=f"Downloading {source_blob_name}") as file_obj:
- blob.download_to_file(file_obj)
-
- print(f"File {source_blob_name} downloaded to {destination_file_name}.")
- except Exception as e:
- print(f"Error downloading file: {e}")
+def download_from_bucket(storage_client: Client, bucket_name: str, bucket_path: str, local_path: str) -> None:
+ """Download the snapshot tar file from Google Cloud Storage."""
+ bucket: Bucket = storage_client.get_bucket(bucket_name)
+ blob = bucket.blob(bucket_path)
+ print(f"Downloading {bucket_path} from gs://{bucket_name}...")
+
+ # Download the tar file to the local path
+ blob.download_to_filename(local_path)
+ print(f"Downloaded {bucket_path} to {local_path}")
-def upload_file_to_bucket(
- storage_client: Client,
- bucket_name: str,
- source_file: str,
- destination_blob_name: str
-) -> None:
- """Upload a file to the specified Google Cloud Storage bucket with progress tracking."""
+def upload_file_to_bucket(storage_client: Client, bucket_name: str, source_file: str, destination_blob_name: str) -> None:
+ """Upload a file to the specified Google Cloud Storage bucket."""
try:
- bucket = storage_client.get_bucket(bucket_name)
+ print(f"Start uploading to {destination_blob_name}.")
+ bucket: Bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(destination_blob_name)
-
- file_size = os.path.getsize(source_file)
-
- with tqdm(total=file_size, unit='B', unit_scale=True, desc=f"Uploading {source_file}") as pbar:
- with open(source_file, "rb") as f:
- chunk_size = 1024 * 1024 * 32
- for chunk in iter(lambda: f.read(chunk_size), b""):
- blob.upload_from_file(io.BytesIO(chunk), content_type="application/octet-stream")
- pbar.update(len(chunk))
-
+ blob.upload_from_filename(source_file)
print(f"File {source_file} uploaded to {destination_blob_name}.")
except Exception as e:
print(f"Error uploading file: {e}")
diff --git a/uv.lock b/uv.lock
index b70b22f..32ee171 100644
--- a/uv.lock
+++ b/uv.lock
@@ -58,15 +58,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767 },
]
-[[package]]
-name = "colorama"
-version = "0.4.6"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 },
-]
-
[[package]]
name = "docker"
version = "7.1.0"
@@ -195,7 +186,6 @@ dependencies = [
{ name = "docker" },
{ name = "google-cloud-storage" },
{ name = "toml" },
- { name = "tqdm" },
]
[package.metadata]
@@ -203,7 +193,6 @@ requires-dist = [
{ name = "docker", specifier = ">=7.1.0" },
{ name = "google-cloud-storage", specifier = ">=2.19.0" },
{ name = "toml", specifier = ">=0.10.2" },
- { name = "tqdm", specifier = ">=4.67.1" },
]
[[package]]
@@ -302,18 +291,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 },
]
-[[package]]
-name = "tqdm"
-version = "4.67.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "colorama", marker = "platform_system == 'Windows'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 },
-]
-
[[package]]
name = "urllib3"
version = "2.3.0"