diff --git a/erna/scripts/check_availability.py b/erna/scripts/check_availability.py index 5dd91dc..541923a 100644 --- a/erna/scripts/check_availability.py +++ b/erna/scripts/check_availability.py @@ -13,13 +13,12 @@ from ..automatic_processing.custom_fields import night_int_to_date from ..utils import load_config -log = logging.getLogger() -log.setLevel(logging.INFO) +log = logging.getLogger('erna') handler = logging.StreamHandler() formatter = logging.Formatter(fmt='%(asctime)s|%(levelname)s|%(name)s|%(message)s') handler.setFormatter(formatter) -log.addHandler(handler) +logging.getLogger().addHandler(handler) db_specification = 'mysql+pymysql://{user}:{password}@{host}/{database}' @@ -38,37 +37,43 @@ def check_availability(run, basedir='/fact/raw'): basedir, str(run.night.year), '{:02d}'.format(run.night.month), '{:02d}'.format(run.night.day), '{:%Y%m%d}_{:03d}'.format(run.night, run.run_id) ) - log.debug('Basename: {}'.format(basename)) - if run.run_type == 1: - f = RawDataFile.select_night_runid(run.night, run.run_id) - available = isfile(basename + '.fits.fz') or isfile(basename + '.fits.gz') - log.debug('Available: {}'.format(available)) - - f.available = available - f.save() - - elif run.run_type == 2 and run.drs_step == 2: + if run.drs_step == 2: log.debug('is a drs file') - f = DrsFile.select_night_runid(run.night, run.run_id) - available = isfile(basename + '.drs.fits.gz') - log.debug('Available: {}'.format(available)) - f.available = available - f.save() + + try: + f = DrsFile.get(night=run.night, run_id=run.run_id) + available = isfile(basename + '.drs.fits.gz') + log.debug('Available: {}'.format(available)) + if available != f.available: + f.available = available + f.save() + except DrsFile.DoesNotExist: + log.info('DrsFile {:%Y%m%d}_{:03d} not in database'.format( + run.night, run.run_id + )) else: - log.debug('Neither drs nor data file') + try: + f = RawDataFile.get(night=run.night, run_id=run.run_id) + available = isfile(basename + '.fits.fz') or isfile(basename + '.fits.gz') + log.debug('Available: {}'.format(available)) + + if available != f.available: + f.available = available + f.save() + except RawDataFile.DoesNotExist: + log.info('Run {:%Y%m%d}_{:03d} not in database'.format( + run.night, run.run_id + )) @click.command() -@click.option('--year', help='The year to update (default all)') -@click.option('--month', help='The month to update (default all)') -@click.option('--day', help='The day to update (default all)') @click.option('--config', '-c', help='Yaml file containing database credentials') @click.option('--verbose', '-v', help='Set logging level to DEBUG', is_flag=True) @click.option('--start', type=parse_date, default=str(datetime.date(2011, 10, 1))) @click.option('--end', type=parse_date, default=str(datetime.date.today())) -def main(year, month, day, config, verbose, start, end): +def main(config, verbose, start, end): ''' Check if RawDataFiles and DrsFiles are available. Goes through the database entries and checks if the file is where it is expected diff --git a/erna/scripts/fill_database.py b/erna/scripts/fill_database.py index eeb3171..94c1853 100644 --- a/erna/scripts/fill_database.py +++ b/erna/scripts/fill_database.py @@ -55,7 +55,6 @@ def main(start, end, config): ) runs['fNight'] = pd.to_datetime(runs.fNight.astype(str), format='%Y%m%d') - # fill all non drs runs into raw_data_files fill_data_runs(runs.query('fDrsStep != 2'), database=database) # fill all drs runs into drs_files