From e9b99b7bd8175f268002f5b1d0f64858eb1f2cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szymaniak?= Date: Sat, 26 Jan 2019 13:25:47 +0100 Subject: [PATCH] AssetManager refactor --- subsync/assets/__init__.py | 43 +----- subsync/assets/downloader.py | 81 ---------- subsync/assets/item.py | 222 ++++++++++++++++++++++++++++ subsync/assets/local.py | 24 --- subsync/assets/mgr.py | 123 ++++++++++++++++ subsync/assets/remote.py | 132 ----------------- subsync/assets/updater.py | 132 ++++++++++++----- subsync/async_utils.py | 49 +++++++ subsync/config.py.template | 2 +- subsync/dictionary.py | 15 +- subsync/gui/downloadwin.py | 228 ++++++++++------------------- subsync/gui/layout/downloadwin.fbp | 1 - subsync/gui/layout/downloadwin.py | 8 - subsync/gui/mainwin.py | 126 ++++++++++------ subsync/speech.py | 27 +--- subsync/thread.py | 38 ++++- subsync/utils.py | 7 + 17 files changed, 707 insertions(+), 551 deletions(-) delete mode 100644 subsync/assets/downloader.py create mode 100644 subsync/assets/item.py delete mode 100644 subsync/assets/local.py create mode 100644 subsync/assets/mgr.py delete mode 100644 subsync/assets/remote.py create mode 100644 subsync/async_utils.py diff --git a/subsync/assets/__init__.py b/subsync/assets/__init__.py index 449696b..e33645c 100644 --- a/subsync/assets/__init__.py +++ b/subsync/assets/__init__.py @@ -1,44 +1,9 @@ -from subsync import config -from subsync import utils -from subsync.assets import local -from subsync.assets import remote -from subsync.assets.downloader import AssetDownloader -from subsync.assets.updater import SelfUpdater +from subsync.assets.mgr import AssetManager -remoteAssets = None +assetManager = AssetManager() -def init(updateCb): - global remoteAssets - remoteAssets = remote.RemoteAssets(updateCb) -def terminate(): - global remoteAssets - if remoteAssets: - remoteAssets.terminate() - remoteAssets = None - -def getLocalAsset(*args, **kwargs): - return local.getAsset(*args, **kwargs) - -def getRemoteAsset(*args, **kwargs): - return remoteAssets.getAsset(*args, **kwargs) - -def isUpdateAvailable(): - if config.assetupd: - currentVer = utils.getCurrentVersion() - remoteUpdate = getRemoteAsset(**config.assetupd) - remoteVer = utils.parseVersion(remoteUpdate.get('version')) - return currentVer and remoteVer and currentVer < remoteVer - -def getAssetPrettyName(type, params, **kw): - if type == 'speech': - return _('{} speech recognition model').format( - utils.getLanguageName(params[0])) - elif type == 'dict': - return _('dictionary {} / {}').format( - utils.getLanguageName(params[0]), - utils.getLanguageName(params[1])) - else: - return '{}/{}'.format(type, '-'.join(params)) +def getAsset(assetId, params=None): + return assetManager.getAsset(assetId, params) diff --git a/subsync/assets/downloader.py b/subsync/assets/downloader.py deleted file mode 100644 index 3454ed8..0000000 --- a/subsync/assets/downloader.py +++ /dev/null @@ -1,81 +0,0 @@ -from subsync import config -from subsync import utils -from subsync import pubkey -from subsync import error -import aiohttp -import tempfile -import zipfile -import Crypto -import os - -import logging -logger = logging.getLogger(__name__) - - -class AssetDownloader(object): - def __init__(self, type=None, url=None, sig=None, version=None, size=None, **kw): - self.type = type - self.url = url - self.sig = sig - self.version = utils.parseVersion(version) - self.size = size - - for key, val in dict(type=type, url=url, sig=sig).items(): - if val == None: - raise error.Error('Invalid asset data, missing parameter', key=key) - - async def download(self, progressCb=None): - logger.info('downloading %s', self.url) - async with aiohttp.ClientSession(read_timeout=None, raise_for_status=True) as session: - async with session.get(self.url) as response: - pos = 0 - size = getSizeFromHeader(response.headers, self.size) - - fp = tempfile.TemporaryFile() - hash = Crypto.Hash.SHA256.new() - - async for chunk, _ in response.content.iter_chunks(): - fp.write(chunk) - hash.update(chunk) - pos += len(chunk) - - if progressCb: - progressCb((pos, size)) - - logger.info('successfully downloaded %s', self.url) - return fp, hash - - async def verify(self, hash): - logger.info('downloading signature') - async with aiohttp.ClientSession() as session: - async with session.get(self.sig) as response: - assert response.status == 200 - sig = await response.read() - - logger.info('verifying signature') - if not pubkey.getVerifier().verify(hash, sig): - raise error.Error(_('Signature verification failed'), url=self.url) - - logger.info('signature is valid') - - async def install(self, fp): - if self.type == 'zip': - dstdir = config.assetdir - logger.info('extracting zip asset to %s', dstdir) - os.makedirs(dstdir, exist_ok=True) - zipf = zipfile.ZipFile(fp) - zipf.extractall(dstdir) - logger.info('extraction completed') - - else: - raise error.Error('Invalid asset type', type=self.type, url=self.url) - - fp.close() - - -def getSizeFromHeader(headers, defaultSize=None): - try: - return int(headers.get('content-length', defaultSize)) - except: - return defaultSize - diff --git a/subsync/assets/item.py b/subsync/assets/item.py new file mode 100644 index 0000000..7869bf2 --- /dev/null +++ b/subsync/assets/item.py @@ -0,0 +1,222 @@ +from subsync.assets.updater import Updater +from subsync import config +from subsync import utils +from subsync.error import Error +import os +import json +import shutil +import subprocess +import stat + +import logging +logger = logging.getLogger(__name__) + + +class Asset(object): + def __init__(self, type, params): + self.type = type + self.params = params + + self.local = None + self.remote = None + self.updater = None + + fname = '{}.{}'.format('-'.join(params), type) + self.path = os.path.join(config.assetdir, type, fname) + self.localDir = None + + def updateLocal(self): + self.local = {} + + def updateRemote(self, remote): + self.remote = remote + + def getPrettyName(self): + return mkId(self.type, self.params) + + def getLocal(self, key=None, defaultValue=None): + if self.local == None: + self.updateLocal() + local = self.local or {} + if key: + return local.get(key, defaultValue) + else: + return local + + def getRemote(self, key=None, defaultValue=None): + remote = self.remote or {} + if key: + return remote.get(key, defaultValue) + else: + return remote + + def getUpdater(self): + if not self.updater and self.remote: + self.updater = Updater(self) + return self.updater + + def isLocal(self): + return bool(self.getLocal()) + + def isRemote(self): + return bool(self.getRemote()) + + def localVersion(self, defaultVersion=(0, 0, 0)): + return utils.parseVersion(self.getLocal('version'), defaultVersion) + + def remoteVersion(self, defaultVersion=(0, 0, 0)): + return utils.parseVersion(self.getRemote('version'), defaultVersion) + + def validateLocal(self): + pass + + def removeLocal(self): + try: + if self.path and os.path.isfile(self.path): + logger.info('removing %s', self.path) + os.remove(self.path) + except Exception as e: + logger.error('cannot remove %s: %r', self.getPrettyName(), e, exc_info=True) + + try: + if self.localDir and os.path.isdir(self.localDir): + logger.info('removing %s', self.localDir) + shutil.rmtree(self.localDir, ignore_errors=True) + except Exception as e: + logger.error('cannot remove %s: %r', self.getPrettyName(), e, exc_info=True) + + self.local = {} + + def isUpgradable(self): + return self.isRemote() and self.remoteVersion() > self.localVersion() + + def __repr__(self): + return ''.format( + mkId(self.type, self.params), + self.getLocal(), + self.isRemote(), + self.path) + + +class DictAsset(Asset): + def updateLocal(self): + try: + with open(self.path, encoding='utf8') as fp: + ents = fp.readline().strip().split('/', 3) + + if len(ents) >= 4 and ents[0] == '#dictionary': + self.local = dict( + lang1 = ents[1], + lang2 = ents[2], + version = ents[3]) + + except Exception as e: + logger.warn('cannot load %s: %r', self.getPrettyName(), e) + self.removeLocal() + + def getPrettyName(self): + if len(self.params) >= 2: + return _('dictionary {} / {}').format( + utils.getLanguageName(self.params[0]), + utils.getLanguageName(self.params[1])) + else: + super().getPrettyName() + + +class SpeechAsset(Asset): + def updateLocal(self): + try: + with open(self.path, encoding='utf8') as fp: + local = json.load(fp) + + # fix local paths + dirname = os.path.abspath(os.path.dirname(self.path)) + sphinx = local.get('sphinx', None) + if sphinx: + for key, val in sphinx.items(): + if val.startswith('./'): + sphinx[key] = os.path.join(dirname, *val.split('/')[1:]) + + localDir = local.get('dir', None) + if localDir and localDir.startswith('./'): + localDir = os.path.join(dirname, *localDir.split('/')[1:]) + + self.local = local + self.localDir = localDir + + except Exception as e: + logger.warn('cannot load %s: %r', self.getPrettyName(), e) + self.removeLocal() + + def getPrettyName(self): + if len(self.params) >= 1: + return _('{} speech recognition model').format( + utils.getLanguageName(self.params[0])) + else: + super().getPrettyName() + + +class UpdateAsset(Asset): + def __init__(self, type, params): + super().__init__(type, params) + self.localDir = os.path.join(config.assetdir, 'upgrade') + self.path = os.path.join(self.localDir, 'upgrade.json') + + def updateLocal(self): + try: + with open(self.path, encoding='utf8') as fp: + self.local = json.load(fp) + + except Exception as e: + logger.warn('cannot load %s: %r', self.getPrettyName(), e) + self.removeLocal() + + def installUpdate(self): + try: + instPath = os.path.join(self.localDir, self.getLocal('install')) + logger.info('executing installer %s', instPath) + mode = os.stat(instPath).st_mode + if (mode & stat.S_IEXEC) == 0: + os.chmod(instPath, mode | stat.S_IEXEC) + subprocess.Popen(instPath, cwd=self.localDir) + + except Exception as e: + logger.error('cannot install update %s: %r', self.path, e, exc_info=True) + raise Error(_('Update instalation failed miserably')) + + def hasUpdate(self): + return self.hasRemoteUpdate() or self.hasLocalUpdate() + + def hasLocalUpdate(self): + cur = utils.getCurrentVersion() + return cur and self.isLocal() and self.localVersion() > cur + + def hasRemoteUpdate(self): + cur = utils.getCurrentVersion() + return cur and self.isRemote() and self.remoteVersion() > cur + + +def getAssetTypeByName(typ, params=None): + types = { + 'dict': DictAsset, + 'speech': SpeechAsset, + 'subsync': UpdateAsset, + } + + T = types.get(typ, Asset) + return T(typ, params) + + +def mkId(type, params): + return '{}/{}'.format(type, '-'.join(params)) + + +def parseId(id): + ents = id.split('/', 1) + if len(ents) == 2: + return ents[0], ents[1].split('-') + elif len(ents) == 1: + return ents[0], None + else: + return None, None + diff --git a/subsync/assets/local.py b/subsync/assets/local.py deleted file mode 100644 index 0346738..0000000 --- a/subsync/assets/local.py +++ /dev/null @@ -1,24 +0,0 @@ -from subsync import assets -from subsync import config -from subsync import error -import os -import itertools - - -def getAsset(type, params, permutable=False, raiseIfMissing=False): - if permutable: - paramsPerm = itertools.permutations(params) - else: - paramsPerm = [ params ] - - for params in paramsPerm: - fname = '{}.{}'.format('-'.join(params), type) - path = os.path.join(config.assetdir, type, fname) - if os.path.isfile(path): - return path - - if raiseIfMissing: - raise error.Error(_('Missing {}').format( - assets.getAssetPrettyName(type, params)), - type=type, params=params) - diff --git a/subsync/assets/mgr.py b/subsync/assets/mgr.py new file mode 100644 index 0000000..8436dfd --- /dev/null +++ b/subsync/assets/mgr.py @@ -0,0 +1,123 @@ +from subsync.assets import item +from subsync import config +from subsync import utils +from subsync import thread +from subsync import async_utils +from subsync.settings import settings +import threading +import asyncio + +import logging +logger = logging.getLogger(__name__) + + +class AssetManager(object): + def __init__(self): + self.assets = {} + self.lock = threading.Lock() + self.updateTask = thread.AsyncJob(self.updateJob, name='AssetsUpdater') + + self.removeOldInstaller() + + async def updateJob(self): + await self.loadRemoteAssetList() + await self.downloadRemoteAssetList() + self.removeOldInstaller() + await self.runAutoUpdater() + + async def loadRemoteAssetList(self): + try: + logger.info('reading remote asset list from %s', config.assetspath) + assets = await async_utils.readJsonFile(config.assetspath) + if assets: + self.updateRemoteAssetsData(assets) + + except asyncio.CancelledError: + raise + + except Exception as e: + logger.error('cannot read asset list from %s: %r', + config.assetspath, e, exc_info=True) + + async def downloadRemoteAssetList(self): + try: + if config.assetsurl: + logger.info('downloading remote assets list from %s', config.assetsurl) + assets = await async_utils.downloadJson(config.assetsurl) + if assets: + await async_utils.writeJsonFile(config.assetspath, assets) + self.updateRemoteAssetsData(assets) + + except asyncio.CancelledError: + raise + + except Exception as e: + logger.error('cannot download asset list from %s: %r', config.assetsurl, e) + + async def runAutoUpdater(self): + try: + updAsset = self.getSelfUpdaterAsset() + updater = updAsset.getUpdater() if updAsset else None + cur = utils.getCurrentVersion() + + if updater and cur: + loc = updAsset.localVersion(None) + rem = updAsset.remoteVersion(None) + + if (loc and cur >= loc) or (loc and rem and rem > loc): + updAsset.removeLocal() + loc = None + + if rem and not loc and rem > cur: + logger.info('new version available to download, %s -> %s', + utils.versionToString(cur), + utils.versionToString(rem)) + + if settings().autoUpdate: + updater.start() + + except asyncio.CancelledError: + raise + + except Exception as e: + logger.error('update processing failed: %r', e, exc_info=True) + + def removeOldInstaller(self): + cur = utils.getCurrentVersion() + if cur: + updAsset = self.getSelfUpdaterAsset() + if updAsset: + loc = updAsset.localVersion(None) + rem = updAsset.remoteVersion(None) + if loc and loc <= cur: + updAsset.removeLocal() + elif loc and rem and loc < rem: + updAsset.removeLocal() + + def getAsset(self, assetId, params=None): + if params: + typ = assetId + par = params + id = item.mkId(typ, par) + elif isinstance(assetId, str): + id = assetId + typ, par = item.parseId(assetId) + else: + typ = assetId[0] + par = assetId[1] + id = item.mkId(typ, par) + + with self.lock: + if id not in self.assets: + self.assets[id] = item.getAssetTypeByName(typ, par) + return self.assets[id] + + def getSelfUpdaterAsset(self): + if config.assetupd: + return self.getAsset(config.assetupd) + + def updateRemoteAssetsData(self, remoteData): + logger.info('update remote asset list, got %i assets', len(remoteData)) + for id, remote in remoteData.items(): + self.getAsset(id).updateRemote(remote) + diff --git a/subsync/assets/remote.py b/subsync/assets/remote.py deleted file mode 100644 index 46e57b4..0000000 --- a/subsync/assets/remote.py +++ /dev/null @@ -1,132 +0,0 @@ -from subsync import assets -from subsync.assets.updater import SelfUpdater -from subsync import config -from subsync import utils -from subsync import error -import json -import asyncio -import aiohttp -import threading -import itertools -import os - -import logging -logger = logging.getLogger(__name__) - - -class RemoteAssets(object): - def __init__(self, updateCb): - self.assets = {} - self.assetsLock = threading.Lock() - - self.loop = None - self.task = None - - def runUpdate(): - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - self.task = asyncio.ensure_future(self.updateJob(updateCb)) - self.loop.run_until_complete(self.task) - self.loop.close() - - self.thread = threading.Thread(name='AssetsUpdate', target=runUpdate) - if config.assetsurl: - self.thread.start() - - def terminate(self): - if self.thread.isAlive(): - self.loop.call_soon_threadsafe(self.task.cancel) - self.thread.join() - - def getAsset(self, type, params, permutable=False, raiseIfMissing=False): - if permutable: - paramsPerm = itertools.permutations(params) - else: - paramsPerm = [ params ] - - for params in paramsPerm: - assetId = '{}/{}'.format(type, '-'.join(params)) - with self.assetsLock: - if assetId in self.assets: - asset = self.assets[assetId] - asset['title'] = assets.getAssetPrettyName(type, params) - return asset - - if raiseIfMissing: - raise error.Error(_('Missing {}').format( - assets.getAssetPrettyName(type, params)), - type=type, params=params) - - async def updateJob(self, updateCb): - try: - await self.loadList() - await self.downloadAssets() - - localUpdate = SelfUpdater.getLocalUpdate() - if localUpdate: - localVer = localUpdate['version'] - currentVer = utils.getCurrentVersion() - logger.info('update version: %s, current version: %s', - localVer, currentVer) - - if currentVer == None or currentVer >= localVer: - SelfUpdater.removeLocalUpdate() - - if updateCb: - updateCb() - - logger.info('update job done') - - except asyncio.CancelledError: - logger.info('update job cancelled') - - async def downloadAssets(self): - try: - logger.info('downloading assets list from %s', config.assetsurl) - async with aiohttp.ClientSession() as session: - async with session.get(config.assetsurl) as response: - assert response.status == 200 - res = await response.json(content_type=None) - - logger.info('got %i assets', len(res)) - - with self.assetsLock: - self.assets = res - - await self.saveList() - - except asyncio.CancelledError: - raise - - except Exception as e: - logger.error('asset list download failed, %r', e, exc_info=True) - - async def loadList(self): - try: - if os.path.isfile(config.assetspath): - with open(config.assetspath, encoding='utf8') as fp: - assets = json.load(fp) - - with self.assetsLock: - self.assets = assets - - except asyncio.CancelledError: - raise - - except Exception as e: - logger.error('cannot load assets list from %s: %r', - config.assetspath, e, exc_info=True) - - async def saveList(self): - try: - os.makedirs(os.path.dirname(config.assetspath), exist_ok=True) - with open(config.assetspath, 'w', encoding='utf8') as fp: - json.dump(self.assets, fp, indent=4) - - except asyncio.CancelledError: - raise - - except Exception as e: - logger.error('cannot write assets list to %s: %r', - config.assetspath, e, exc_info=True) - diff --git a/subsync/assets/updater.py b/subsync/assets/updater.py index 042c54f..c8a548e 100644 --- a/subsync/assets/updater.py +++ b/subsync/assets/updater.py @@ -1,55 +1,109 @@ from subsync import config -from subsync import utils -from subsync import error -import json -import shutil +from subsync import thread +from subsync import pubkey +from subsync import async_utils +from subsync.error import Error import os -import subprocess -import stat +import sys +import threading +import asyncio +import tempfile +import zipfile +import Crypto import logging logger = logging.getLogger(__name__) -class SelfUpdater(object): - def getLocalUpdate(): - try: - dirPath = os.path.join(config.assetdir, 'upgrade') - if os.path.isdir(dirPath): - path = os.path.join(dirPath, 'upgrade.json') - with open(path, encoding='utf8') as fp: - update = json.load(fp) +class Updater(thread.AsyncJob): + def __init__(self, asset): + self.asset = asset - return dict( - version = utils.parseVersion(update['version']), - path = os.path.join(dirPath, update['install']), - cwd = dirPath) + self.lock = threading.Lock() + self.done = False + self.progress = 0 + self.error = None - except Exception as e: - logger.warning('read update info failed, %r', e, exc_info=True) - removeLocalUpdate() + super().__init__(self.job, name='Download') - def removeLocalUpdate(): - try: - path = os.path.join(config.assetdir, 'upgrade') - if os.path.isdir(path): - logger.info('removing update from %s', path) - shutil.rmtree(path, ignore_errors=True) + for key in ['type', 'url', 'sig']: + if key not in asset.getRemote(): + raise Error('Invalid asset data, missing parameter', key=key) - except Exception as e: - logger.error('cannot remove upgrade data from %s: %r', path, e) + def start(self): + self.setState(done=False, progress=0.0, error=None) + super().start() + + def setState(self, **kw): + with self.lock: + for key, value in kw.items(): + setattr(self, key, value) - def installLocalUpdate(): + def getState(self): + with self.lock: + return self.done, self.progress, self.error + + async def job(self): + logger.info('downloading asset %s', self.asset.getPrettyName()) try: - update = getLocalUpdate() - path = update['path'] + with tempfile.TemporaryFile() as fp: + fp, hash = await self.download(fp) + self.setState(progress=1.0) + await self.verify(hash) + self.asset.removeLocal() + await self.install(fp) + self.asset.updateLocal() - logger.info('executing installer %s', path) - mode = os.stat(path).st_mode - if (mode & stat.S_IEXEC) == 0: - os.chmod(path, mode | stat.S_IEXEC) - subprocess.Popen(path, cwd=update['cwd']) + except asyncio.CancelledError: + logger.info('operation cancelled by user') + self.asset.removeLocal() + self.setState(result=False) except Exception as e: - logger.error('cannot install update %s: %r', path, e) - raise error.Error(_('Update instalation failed miserably')) + logger.error('download failed, %r', e, exc_info=True) + self.asset.removeLocal() + self.setState(error=sys.exc_info()) + + finally: + self.setState(done=True) + + async def download(self, fp): + url = self.asset.getRemote('url') + size = self.asset.getRemote('size') + + logger.info('downloading %s', url) + hash = Crypto.Hash.SHA256.new() + + def onNewChunk(chunk, progress): + self.setState(progress=progress) + hash.update(chunk) + + await async_utils.downloadFileProgress(url, fp, size, chunkCb=onNewChunk) + return fp, hash + + async def verify(self, hash): + logger.info('downloading signature') + sig = await async_utils.downloadRaw(self.asset.getRemote('sig')) + + logger.info('verifying signature') + if not pubkey.getVerifier().verify(hash, sig): + raise Error(_('Signature verification failed'), + url=self.asset.getRemote('url')) + + logger.info('signature is valid') + + async def install(self, fp): + assetType = self.asset.getRemote('type') + if assetType == 'zip': + dstdir = config.assetdir + logger.info('extracting zip asset to %s', dstdir) + os.makedirs(dstdir, exist_ok=True) + zipf = zipfile.ZipFile(fp) + zipf.extractall(dstdir) + logger.info('extraction completed') + + else: + raise Error('Invalid asset type', + type=assetType, + url=self.asset.getRemote('url')) + diff --git a/subsync/async_utils.py b/subsync/async_utils.py new file mode 100644 index 0000000..0740d79 --- /dev/null +++ b/subsync/async_utils.py @@ -0,0 +1,49 @@ +import os +import aiohttp +import json + +import logging +logger = logging.getLogger(__name__) + + +async def downloadRaw(url): + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + assert response.status == 200 + return await response.read() + + +async def downloadJson(url): + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + assert response.status == 200 + return await response.json(content_type=None) + + +async def downloadFileProgress(url, fp, size=None, chunkCb=None): + async with aiohttp.ClientSession(read_timeout=None, raise_for_status=True) as session: + async with session.get(url) as response: + pos = 0 + try: + size = int(response.headers.get('content-length', size)) + except: + pass + + async for chunk, _ in response.content.iter_chunks(): + fp.write(chunk) + pos += len(chunk) + + if chunkCb: + chunkCb(chunk, (pos, size)) + + +async def readJsonFile(path): + if os.path.isfile(path): + with open(path, encoding='utf8') as fp: + return json.load(fp) + + +async def writeJsonFile(path, data): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w', encoding='utf8') as fp: + json.dump(data, fp, indent=4) diff --git a/subsync/config.py.template b/subsync/config.py.template index 6cd7543..761e329 100644 --- a/subsync/config.py.template +++ b/subsync/config.py.template @@ -7,7 +7,7 @@ datadir = os.path.dirname(__file__) if sys.platform == 'win32': configdir = os.path.join(os.environ['APPDATA'], appname) shareddir = os.path.join(os.environ['ALLUSERSPROFILE'], appname) - assetupd = dict(type='subsync', params=('win', 'x86_64')) + assetupd = 'subsync/win-x86_64' elif sys.platform == 'linux': configdir = os.path.join(os.path.expanduser('~'), '.config', appname) diff --git a/subsync/dictionary.py b/subsync/dictionary.py index 4d1f5ee..0f9b547 100644 --- a/subsync/dictionary.py +++ b/subsync/dictionary.py @@ -1,7 +1,6 @@ import gizmo from subsync import assets from subsync.error import Error -import os import logging logger = logging.getLogger(__name__) @@ -10,20 +9,20 @@ def loadDictionary(lang1, lang2, minLen=0): dictionary = gizmo.Dictionary() - dictPath = assets.getLocalAsset('dict', (lang1, lang2)) - if dictPath: - for key, val in loadDictionaryFromFile(dictPath): + asset = assets.getAsset('dict', (lang1, lang2)) + if asset.isLocal(): + for key, val in loadDictionaryFromFile(asset.path): if len(key) >= minLen and len(val) >= minLen: dictionary.add(key, val) else: - dictPath = assets.getLocalAsset('dict', (lang2, lang1)) - if dictPath: - for key, val in loadDictionaryFromFile(dictPath): + asset = assets.getAsset('dict', (lang2, lang1)) + if asset.isLocal(): + for key, val in loadDictionaryFromFile(asset.path): if len(key) >= minLen and len(val) >= minLen: dictionary.add(val, key) - if not dictPath: + if not asset.isLocal(): raise Error(_('There is no dictionary for transaltion from {} to {}') .format(lang1, lang2)) \ .add('language1', lang1) \ diff --git a/subsync/gui/downloadwin.py b/subsync/gui/downloadwin.py index 35a718b..da272ea 100644 --- a/subsync/gui/downloadwin.py +++ b/subsync/gui/downloadwin.py @@ -1,194 +1,116 @@ import subsync.gui.layout.downloadwin from subsync.gui import errorwin import wx -import sys -import asyncio -import threading -from subsync import config -from subsync import assets -from subsync.assets import SelfUpdater, AssetDownloader +from subsync.assets import assetManager from subsync import thread -from subsync.settings import settings -from subsync.utils import fileSizeFmt +from subsync import utils +from subsync.error import Error import logging logger = logging.getLogger(__name__) class DownloadWin(subsync.gui.layout.downloadwin.DownloadWin): - def __init__(self, parent, title, job): + def __init__(self, parent, title, updater): super().__init__(parent) self.m_textName.SetLabel(title) - self.loop = None - self.task = None - - self.progress = thread.AtomicValue(None) - self.lastPos = 0 + self.updater = updater + self.lastPos = None self.progressTimer = wx.Timer(self) self.Bind(wx.EVT_TIMER, self.onProgressTimerTick, self.progressTimer) - self.progressTimer.Start(500) + self.progressTimer.Start(200) - self.thread = threading.Thread(name='Download', target=self.run, args=[job]) - self.thread.start() + def ShowModal(self): + res = super().ShowModal() - def stop(self): + self.updater.stop() if self.progressTimer.IsRunning(): self.progressTimer.Stop() - if self.loop: - self.loop.call_soon_threadsafe(self.task.cancel) - if self.thread.isAlive(): - self.thread.join() - def onButtonCancelClick(self, event): - self.stop() + return res def onProgressTimerTick(self, event): - progress = self.progress.get() - if progress: - interval = self.progressTimer.GetInterval() / 1000.0 - self.setStatus(*progress, interval) - - @thread.gui_thread - def EndModal(self, retCode): - if self.IsModal(): - return super().EndModal(retCode) - - def run(self, job): - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - self.task = asyncio.ensure_future(self.jobWrapper(job)) - self.loop.run_until_complete(self.task) - - async def jobWrapper(self, job): - try: - await job - - except asyncio.CancelledError: - logger.info('operation cancelled by user') - self.EndModal(wx.ID_CANCEL) - - except Exception as e: - logger.error('download failed, %r', e, exc_info=True) - self.setStatus(_('operation failed')) + done, progress, error = self.updater.getState() - @thread.gui_thread - def showExceptionDlgAndQuit(excInfo): - if self.IsModal(): - errorwin.showExceptionDlg(self, excInfo=excInfo, - msg=_('Operation failed')) - self.EndModal(wx.ID_CANCEL) + if done: + if self.progressTimer.IsRunning(): + self.progressTimer.Stop() - showExceptionDlgAndQuit(sys.exc_info()) + if error: + self.setStatus(_('operation failed')) + errorwin.showExceptionDlg(self, error) + res = wx.ID_CANCEL - finally: - @thread.gui_thread - def stopTimerIfRunning(): - if self.progressTimer.IsRunning(): - self.progressTimer.Stop() - - stopTimerIfRunning() - - async def downloadJob(self, asset): - downloader = AssetDownloader(**asset) - - self.setStatus(_('downloading...')) - fp, hash = await downloader.download(lambda progress: - self.progress.set((_('downloading'), progress))) - self.progress.set(None) + else: + self.setProgress(1.0) + self.setStatus(_('operation finished successfully')) + res = wx.ID_OK - self.setProgress(1) - self.setStatus(_('verifying...')) - await downloader.verify(hash) + wx.Yield() - self.setStatus(_('processing...')) - await downloader.install(fp) - self.setStatus(_('done')) + if self.IsModal(): + self.EndModal(res) - @thread.gui_thread - def setStatus(self, status, progress=None, interval=None): - if progress: + elif isinstance(progress, tuple): pos, size = progress + self.setStatus(_('downloading'), pos, size) if size: - self.m_gaugeProgress.SetValue(min(100, int(100.0 * pos / size))) - msg = '{} {} / {}'.format(status, fileSizeFmt(pos), fileSizeFmt(size)) + self.setProgress(pos / size) else: - self.m_gaugeProgress.Pulse() - msg = '{} {}'.format(status, fileSizeFmt(pos)) - - if interval: - if self.lastPos: - delta = pos - self.lastPos - if delta > 0: - msg += ' ({}/s)'.format(fileSizeFmt(delta / interval)) - self.lastPos = pos + self.setProgress(None) + + elif progress != None: + self.setStatus(_('processing...')) + self.setProgress(progress) + + def setStatus(self, desc, pos=None, size=None): + msg = [ desc ] + if pos != None: + msg += [ utils.fileSizeFmt(pos) ] + if size != None: + msg += [ '/', utils.fileSizeFmt(size) ] + msg += [ self.getDownloadSpeed(pos) ] + self.m_textDetails.SetLabel(' '.join(msg)) + + def setProgress(self, progress): + if progress == None: + self.m_gaugeProgress.Pulse() else: - msg = status - self.m_textDetails.SetLabel(msg) + p = max(min(progress, 1.0), 0.0) + self.m_gaugeProgress.SetValue(int(100.0 * p)) + + def getDownloadSpeed(self, pos): + res = '' + if pos != None and self.lastPos != None: + delta = pos - self.lastPos + if delta > 0: + interval = self.progressTimer.GetInterval() / 1000.0 + res = '({}/s)'.format(utils.fileSizeFmt(delta / interval)) + self.lastPos = pos + return res @thread.gui_thread - def setProgress(self, val): - self.m_gaugeProgress.SetValue(min(100, int(100.0 * val))) - - -class AssetDownloadWin(DownloadWin): - def __init__(self, parent, asset): - super().__init__(parent, asset['title'], job=self.downloadAssetJob(asset)) - - async def downloadAssetJob(self, asset): - await self.downloadJob(asset) - self.EndModal(wx.ID_OK) + def onUpdateComplete(self, upd, success): + if success: + self.setProgress(1.0) + self.setStatus('operation finished successfully') + if self.IsModal(): + self.EndModal(wx.ID_OK) + else: + self.setStatus('operation failed') + if self.IsModal(): + self.EndModal(wx.ID_CANCEL) -class SelfUpdaterWin(DownloadWin): +class SelfUpdateWin(DownloadWin): def __init__(self, parent): title = _('Application upgrade') - super().__init__(parent, title, job=self.updateJob()) - - def onButtonCancelClick(self, event): - self.EndModal(wx.ID_CLOSE) - - async def updateJob(self): - logger.info('new version is available for update') - if not SelfUpdater.getLocalUpdate(): - asset = assets.getRemoteAsset(**config.assetupd) - await self.downloadJob(asset) + asset = assetManager.getSelfUpdaterAsset() + updater = asset.getUpdater() if asset else None + if not updater: + raise Error('Application upgrade is not available') - self.setProgress(1) - self.setStatus(_('update ready')) - - @thread.gui_thread - def askForUpdateIfVisible(): - if self.IsModal(): - self.askForUpdate(self) - - askForUpdateIfVisible() - - @errorwin.error_dlg - def startUpdate(self, parent, ask=True): - if self.thread.is_alive(): - return super().ShowModal() == wx.ID_OK - elif ask: - return self.askForUpdate(parent) - else: - SelfUpdater.installLocalUpdate() - return True - - @errorwin.error_dlg - def askForUpdate(self, parent=None): - dlg = wx.MessageDialog( - parent if parent else self, - _('New version is ready to be installed. Upgrade now?'), - _('Upgrade'), - wx.YES_NO | wx.YES_DEFAULT | wx.ICON_QUESTION) - - if dlg.ShowModal() == wx.ID_YES: - self.EndModal(wx.ID_OK) - SelfUpdater.installLocalUpdate() - return True - - else: - self.EndModal(wx.ID_CANCEL) - return False + super().__init__(parent, title, updater) diff --git a/subsync/gui/layout/downloadwin.fbp b/subsync/gui/layout/downloadwin.fbp index e1ec8f8..1e88be2 100644 --- a/subsync/gui/layout/downloadwin.fbp +++ b/subsync/gui/layout/downloadwin.fbp @@ -595,7 +595,6 @@ - onButtonCancelClick diff --git a/subsync/gui/layout/downloadwin.py b/subsync/gui/layout/downloadwin.py index 74d256a..a0fd5f9 100644 --- a/subsync/gui/layout/downloadwin.py +++ b/subsync/gui/layout/downloadwin.py @@ -71,16 +71,8 @@ def __init__( self, parent ): bSizer1.Fit( self ) self.Centre( wx.BOTH ) - - # Connect Events - self.m_buttonCancel.Bind( wx.EVT_BUTTON, self.onButtonCancelClick ) def __del__( self ): pass - - # Virtual event handlers, overide them in your derived class - def onButtonCancelClick( self, event ): - event.Skip() - diff --git a/subsync/gui/mainwin.py b/subsync/gui/mainwin.py index 68b0756..993dcef 100644 --- a/subsync/gui/mainwin.py +++ b/subsync/gui/mainwin.py @@ -2,13 +2,13 @@ import wx from subsync.gui.syncwin import SyncWin from subsync.gui.settingswin import SettingsWin -from subsync.gui.downloadwin import AssetDownloadWin, SelfUpdaterWin +from subsync.gui.downloadwin import DownloadWin, SelfUpdateWin from subsync.gui.aboutwin import AboutWin +from subsync.gui.busydlg import BusyDlg from subsync.gui.errorwin import error_dlg -from subsync import assets +from subsync.assets import assetManager from subsync import cache from subsync import img -from subsync import thread from subsync import config from subsync import loggercfg from subsync.settings import settings @@ -66,14 +66,7 @@ def __init__(self, parent, subs=None, refs=None): self.Layout() self.refsCache = cache.WordsCache() - - self.selfUpdater = None - assets.init(self.assetsUpdated) - - @thread.gui_thread - def assetsUpdated(self): - if settings().autoUpdate and assets.isUpdateAvailable(): - self.selfUpdater = SelfUpdaterWin(self) + assetManager.updateTask.start() def onSliderMaxDistScroll(self, event): val = self.m_sliderMaxDist.GetValue() @@ -113,12 +106,19 @@ def changeSettings(self, newSettings): def onMenuItemCheckUpdateClick(self, event): - if self.selfUpdater == None and assets.isUpdateAvailable(): - self.selfUpdater = SelfUpdaterWin(self) + updAsset = assetManager.getSelfUpdaterAsset() + hasLocalUpdate = updAsset and updAsset.hasLocalUpdate() + + if not assetManager.updateTask.isRunning() and not hasLocalUpdate: + assetManager.updateTask.start() - if self.selfUpdater: - if self.selfUpdater.startUpdate(self): - self.Close(force=True) + if assetManager.updateTask.isRunning(): + with BusyDlg(_('Checking for update...')): + while assetManager.updateTask.isRunning(): + wx.Yield() + + if self.runUpdater(): + self.Close(force=True) else: dlg = wx.MessageDialog( @@ -176,49 +176,93 @@ def validateAssets(self): needAssets = [] if refs.type == 'audio': - needAssets.append(dict(type='speech', params=[refs.lang])) + needAssets.append(assetManager.getAsset('speech', [refs.lang])) if subs.lang and refs.lang and subs.lang != refs.lang: - needAssets.append(dict(type='dict', params=[subs.lang, refs.lang], - permutable=True)) + langs = sorted([subs.lang, refs.lang]) + needAssets.append(assetManager.getAsset('dict', langs)) + + missingAssets = [ a for a in needAssets if not a.isLocal() ] + downloadAssets = [ a for a in missingAssets if a.isRemote() ] + if downloadAssets: + if not self.askForDownloadAssets(downloadAssets): + return False - missingAssets = [ a for a in needAssets if assets.getLocalAsset(**a) == None ] - if len(missingAssets) == 0: - return True + updateAssets = [ a for a in needAssets if a.remoteVersion() > a.localVersion() ] + if updateAssets: + self.askForUpdateAssets(updateAssets) - downloadAssets = [] - for id in missingAssets: - asset = assets.getRemoteAsset(**id, raiseIfMissing=True) - downloadAssets.append(asset) + return True - msg = _('Following assets must be download to continue:\n') - msg += '\n'.join([' - ' + asset['title'] for asset in downloadAssets]) - msg += '\n\n' + _('Download now?') + def askForDownloadAssets(self, assetList): + msg = [ _('Following assets must be download to continue:') ] + msg += [ ' - ' + a.getPrettyName() for a in assetList ] + msg += [ '', ('Download now?') ] title = _('Download assets') - with wx.MessageDialog(self, msg, title, wx.YES_NO | wx.ICON_QUESTION) as dlg: + flags = wx.YES_NO | wx.ICON_QUESTION + with wx.MessageDialog(self, '\n'.join(msg), title, flags) as dlg: if dlg.ShowModal() == wx.ID_YES: - return self.downloadAssets(downloadAssets) - - def downloadAssets(self, assetsl): - for asset in assetsl: - with AssetDownloadWin(self, asset) as dlg: + return self.downloadAssets(assetList) + return False + + def askForUpdateAssets(self, assetList): + msg = [ _('Following assets could be updated:') ] + msg += [ ' - ' + a.getPrettyName() for a in assetList ] + msg += [ '', ('Update now?') ] + title = _('Update assets') + flags = wx.YES_NO | wx.ICON_QUESTION + with wx.MessageDialog(self, '\n'.join(msg), title, flags) as dlg: + if dlg.ShowModal() == wx.ID_YES: + self.refsCache.clear() + return self.downloadAssets(assetList) + return False + + def downloadAssets(self, assetList): + for asset in assetList: + upd = asset.getUpdater() + if not upd: + return False + + upd.start() + with DownloadWin(self, asset.getPrettyName(), upd) as dlg: if dlg.ShowModal() != wx.ID_OK: return False return True @error_dlg def onClose(self, event): - if self.selfUpdater: - if event.CanVeto() and settings().askForUpdate: + if event.CanVeto() and settings().askForUpdate: + updAsset = assetManager.getSelfUpdaterAsset() + + if updAsset and updAsset.hasUpdate(): dlg = wx.MessageDialog( self, _('New version is available. Update now?'), _('Upgrade'), wx.YES_NO | wx.YES_DEFAULT | wx.ICON_QUESTION) - if dlg.ShowModal() == wx.ID_YES: - self.selfUpdater.startUpdate(self, ask=False) - self.selfUpdater.stop() + if dlg.ShowModal() == wx.ID_YES: + self.runUpdater(False) - assets.terminate() + assetManager.updateTask.stop() event.Skip() + def runUpdater(self, askForUpdate=True): + updAsset = assetManager.getSelfUpdaterAsset() + if updAsset and updAsset.hasUpdate(): + if not updAsset.hasLocalUpdate(): + SelfUpdateWin(self).ShowModal() + + if askForUpdate: + dlg = wx.MessageDialog( + parent if parent else self, + _('New version is ready to be installed. Upgrade now?'), + _('Upgrade'), + wx.YES_NO | wx.YES_DEFAULT | wx.ICON_QUESTION) + if dlg.ShowModal != wx.ID_YES: + return False + + if updAsset.hasLocalUpdate(): + updAsset.installUpdate() + return True + return False + diff --git a/subsync/speech.py b/subsync/speech.py index da87017..ea7a985 100644 --- a/subsync/speech.py +++ b/subsync/speech.py @@ -1,36 +1,21 @@ import gizmo from subsync import assets from subsync import error -import json -import os import logging logger = logging.getLogger(__name__) -_speechModels = {} - def loadSpeechModel(lang): - if lang in _speechModels: - return _speechModels[lang] - logger.info('loading speech recognition model for language %s', lang) - path = assets.getLocalAsset('speech', [lang], raiseIfMissing=True) - with open(path, encoding='utf8') as fp: - model = json.load(fp) + asset = assets.getAsset('speech', [lang]) + if asset.isLocal(): + logger.debug('model ready: %s', asset.getLocal()) + return asset.getLocal() - # fix paths - if 'sphinx' in model: - dirname = os.path.abspath(os.path.dirname(path)) - sphinx = model['sphinx'] - for key, val in sphinx.items(): - if val.startswith('./'): - sphinx[key] = os.path.join(dirname, *val.split('/')[1:]) - - logger.debug('model ready: %s', model) - _speechModels[lang] = model - return model + raise error.Error(_('There is no speech recognition model for language {}') + .format(lang)).add('language', lang) def createSpeechRec(model): diff --git a/subsync/thread.py b/subsync/thread.py index 4313563..668a7f2 100644 --- a/subsync/thread.py +++ b/subsync/thread.py @@ -1,9 +1,8 @@ import wx from functools import wraps import threading - -import logging -logger = logging.getLogger(__name__) +import asyncio +import aiohttp class AtomicValue(object): @@ -77,3 +76,36 @@ def wrapper(self, *args, **kwargs): return wrapper return decorator + +class AsyncJob(object): + def __init__(self, job, name=None): + self.loop = None + self.task = None + self.thread = None + self.params = dict(name=name, args=[job]) + + def start(self): + thread = threading.Thread(**self.params, target=self._run) + thread.start() + self.thread = thread + + def stop(self): + if self.loop: + self.loop.call_soon_threadsafe(self.task.cancel) + if self.thread and self.thread.isAlive(): + self.thread.join() + + def isRunning(self): + return self.thread and self.thread.isAlive() + + def _run(self, job): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + self.task = asyncio.ensure_future(job()) + self.loop.run_until_complete(self.task) + + +def repeateCancelledError(e): + if isinstance(e, asyncio.CancelledError): + raise + diff --git a/subsync/utils.py b/subsync/utils.py index 298f596..cec8fce 100644 --- a/subsync/utils.py +++ b/subsync/utils.py @@ -15,6 +15,13 @@ def parseVersion(version, defaultVer=None): return defaultVer +def versionToString(version, defaultVer=None): + try: + return '.'.join([ str(v) for v in version ]) + except: + return defaultVer + + def getCurrentVersion(defaultVer=None): try: from subsync.version import version_short