From 5b670ff5e84ebb06087d4dce4ffc2a29d5829eb4 Mon Sep 17 00:00:00 2001 From: Nikita Panyuhin Date: Sun, 17 Dec 2023 00:03:20 +0100 Subject: [PATCH] Removed image downloading logic --- README.md | 2 +- src/Region.py | 6 +----- src/bing.py | 16 +++++++++++----- src/scripts/upload_images.py | 9 +++++---- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4956168d..9e1268c9 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Stages (roughly in order of importance): - [x] Upload alll images to storage - [x] Replace spaces by `\t` in api to reduce space - [ ] Fix metadata for all images (currently done: ?/?) -- [ ] Finally remove all images from this repo and reduce the size of repo (+ number of commits in repo) +- [x] Finally remove all images from this repo and reduce the size of repo (+ number of commits in repo) - [ ] Write a comprehensive README - [ ] Enable other countries - [ ] Improve website diff --git a/src/Region.py b/src/Region.py index 24838f6f..f3b64b64 100644 --- a/src/Region.py +++ b/src/Region.py @@ -9,12 +9,8 @@ def __init__(self, region): self.lang, self.country = map(str.lower, region.split('-')) self.path = mkpath(os.path.dirname(__file__), '../api', self.country.upper()) self.api_path = mkpath(self.path, self.country.lower() + '.json') - self.images_path = mkpath(self.path, 'images') - os.makedirs(self.images_path, exist_ok=True) - - self.relative_path = self.country.upper() - self.relative_images_path = mkpath(self.relative_path, 'images') + self.gcloud_images_path = mkpath(self.country.upper(), 'images') @property def mkt(self): diff --git a/src/bing.py b/src/bing.py index 2bd74190..baa0381b 100644 --- a/src/bing.py +++ b/src/bing.py @@ -1,4 +1,5 @@ from datetime import datetime +import os import requests @@ -62,8 +63,6 @@ def update(region: Region): common_params = {'mkt': region.mkt, 'setlang': region.lang, 'cc': region.country} - # os.makedirs(region.images_path, exist_ok=True) - api_by_date = {item['date']: item for item in region.read_api()} to_download = set() @@ -151,15 +150,22 @@ def update(region: Region): # ------------------------------------------------------------------------------------------------------------------ print('Downloading images and uploading to Google Cloud...') + os.makedirs('_temp', exist_ok=True) + for date in sorted(to_download): filename = date + '.jpg' - image_path = mkpath(region.images_path, filename) - api_by_date[date]['path'] = posixpath(mkpath(region.relative_images_path, filename)) + image_path = mkpath('_temp', filename) with open(image_path, 'wb') as file: file.write(requests.get(api_by_date[date]['bing_url']).content) - api_by_date[date]['url'] = gcloud.upload_file(image_path, api_by_date[date]['path'], skip_exists=True) + api_by_date[date]['url'] = gcloud.upload_file( + image_path, posixpath(mkpath(region.gcloud_images_path, filename)), skip_exists=True + ) + + os.remove(image_path) + + os.rmdir('_temp') # ------------------------------------------------------------------------------------------------------------------ diff --git a/src/scripts/upload_images.py b/src/scripts/upload_images.py index c8b96c4f..eff297c4 100644 --- a/src/scripts/upload_images.py +++ b/src/scripts/upload_images.py @@ -2,9 +2,9 @@ import os sys.path.append('../') +from utils import mkpath, posixpath # noqa: E402 from Region import REGIONS # noqa: E402 from gcloud import GCloud # noqa: E402 -from utils import mkpath # noqa: E402 def main(): @@ -14,9 +14,10 @@ def main(): api = region.read_api() for item in api: - image_path = mkpath(region.images_path, item['date'] + '.jpg') - if os.path.isfile(image_path): - item['url'] = gcloud.upload_file(image_path, item['path'], skip_exists=True) + if os.path.isfile(item['path']): + item['url'] = gcloud.upload_file( + item['path'], posixpath(mkpath(region.gcloud_images_path, item['date'] + '.jpg')), skip_exists=True + ) region.write_api(api)