From 3af32c5451e054b9418ebd20bf24af2047009ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Thu, 4 May 2017 13:45:42 +0200 Subject: [PATCH 1/3] Adapt check availability script to new database scheme --- erna/scripts/check_availability.py | 34 +++++++++++++++++------------- erna/scripts/fill_database.py | 1 - 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/erna/scripts/check_availability.py b/erna/scripts/check_availability.py index 5dd91dc..921eb89 100644 --- a/erna/scripts/check_availability.py +++ b/erna/scripts/check_availability.py @@ -40,24 +40,28 @@ def check_availability(run, basedir='/fact/raw'): ) log.debug('Basename: {}'.format(basename)) - if run.run_type == 1: - f = RawDataFile.select_night_runid(run.night, run.run_id) - available = isfile(basename + '.fits.fz') or isfile(basename + '.fits.gz') - log.debug('Available: {}'.format(available)) - - f.available = available - f.save() - - elif run.run_type == 2 and run.drs_step == 2: + if run.drs_step == 2: log.debug('is a drs file') - f = DrsFile.select_night_runid(run.night, run.run_id) - available = isfile(basename + '.drs.fits.gz') - log.debug('Available: {}'.format(available)) - f.available = available - f.save() + + try: + f = DrsFile.get(night=run.night, run=run.run_id) + available = isfile(basename + '.drs.fits.gz') + log.debug('Available: {}'.format(available)) + f.available = available + f.save() + except DrsFile.DoesNotExist: + log.info('Run not not erna database') else: - log.debug('Neither drs nor data file') + try: + f = RawDataFile.get(night=run.night, run=run.run_id) + available = isfile(basename + '.fits.fz') or isfile(basename + '.fits.gz') + log.debug('Available: {}'.format(available)) + + f.available = available + f.save() + except RawDataFile.DoesNotExist: + log.info('Run not not erna database') @click.command() diff --git a/erna/scripts/fill_database.py b/erna/scripts/fill_database.py index eeb3171..94c1853 100644 --- a/erna/scripts/fill_database.py +++ b/erna/scripts/fill_database.py @@ -55,7 +55,6 @@ def main(start, end, config): ) runs['fNight'] = pd.to_datetime(runs.fNight.astype(str), format='%Y%m%d') - # fill all non drs runs into raw_data_files fill_data_runs(runs.query('fDrsStep != 2'), database=database) # fill all drs runs into drs_files From 99a6f1c0e9929e0ac5f8a6bb80f70699863dae11 Mon Sep 17 00:00:00 2001 From: Maximilian Noethe Date: Thu, 4 May 2017 19:25:17 +0200 Subject: [PATCH 2/3] Fix check_availability --- erna/scripts/check_availability.py | 33 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/erna/scripts/check_availability.py b/erna/scripts/check_availability.py index 921eb89..541923a 100644 --- a/erna/scripts/check_availability.py +++ b/erna/scripts/check_availability.py @@ -13,13 +13,12 @@ from ..automatic_processing.custom_fields import night_int_to_date from ..utils import load_config -log = logging.getLogger() -log.setLevel(logging.INFO) +log = logging.getLogger('erna') handler = logging.StreamHandler() formatter = logging.Formatter(fmt='%(asctime)s|%(levelname)s|%(name)s|%(message)s') handler.setFormatter(formatter) -log.addHandler(handler) +logging.getLogger().addHandler(handler) db_specification = 'mysql+pymysql://{user}:{password}@{host}/{database}' @@ -38,41 +37,43 @@ def check_availability(run, basedir='/fact/raw'): basedir, str(run.night.year), '{:02d}'.format(run.night.month), '{:02d}'.format(run.night.day), '{:%Y%m%d}_{:03d}'.format(run.night, run.run_id) ) - log.debug('Basename: {}'.format(basename)) if run.drs_step == 2: log.debug('is a drs file') try: - f = DrsFile.get(night=run.night, run=run.run_id) + f = DrsFile.get(night=run.night, run_id=run.run_id) available = isfile(basename + '.drs.fits.gz') log.debug('Available: {}'.format(available)) - f.available = available - f.save() + if available != f.available: + f.available = available + f.save() except DrsFile.DoesNotExist: - log.info('Run not not erna database') + log.info('DrsFile {:%Y%m%d}_{:03d} not in database'.format( + run.night, run.run_id + )) else: try: - f = RawDataFile.get(night=run.night, run=run.run_id) + f = RawDataFile.get(night=run.night, run_id=run.run_id) available = isfile(basename + '.fits.fz') or isfile(basename + '.fits.gz') log.debug('Available: {}'.format(available)) - f.available = available - f.save() + if available != f.available: + f.available = available + f.save() except RawDataFile.DoesNotExist: - log.info('Run not not erna database') + log.info('Run {:%Y%m%d}_{:03d} not in database'.format( + run.night, run.run_id + )) @click.command() -@click.option('--year', help='The year to update (default all)') -@click.option('--month', help='The month to update (default all)') -@click.option('--day', help='The day to update (default all)') @click.option('--config', '-c', help='Yaml file containing database credentials') @click.option('--verbose', '-v', help='Set logging level to DEBUG', is_flag=True) @click.option('--start', type=parse_date, default=str(datetime.date(2011, 10, 1))) @click.option('--end', type=parse_date, default=str(datetime.date.today())) -def main(year, month, day, config, verbose, start, end): +def main(config, verbose, start, end): ''' Check if RawDataFiles and DrsFiles are available. Goes through the database entries and checks if the file is where it is expected From 36cd9f7a98ce8149effe6cd2ddaa4e39b354f3a4 Mon Sep 17 00:00:00 2001 From: Maximilian Noethe Date: Thu, 4 May 2017 19:26:52 +0200 Subject: [PATCH 3/3] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c736e8..f0c60a0 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='erna', - version='0.2.0', + version='0.2.1', description='Easy RuN Access. Tools that help to do batch processing of FACT data', url='https://github.com/fact-project/erna', author='Kai Brügge, Jens Buss, Maximilian Nöthe',