Add new option to run just new jobs or jobs without history

Rename option to prepare-jobs
Change Exit Code to 0 to indicate a success
new_jobs was out of scope and just the last job was added
Added has_history_data() it queues the database just once and cache the result for future use.

lib/urlwatch/command.py

@@ -142,6 +142,19 @@ def test_filter(self, id):
         # (ignore_cached) and we do not want to store the newly-retrieved data yet (filter testing)
         return 0
 
+    def prepare_jobs(self):
+        new_jobs = []
+        for idx, job in enumerate(self.urlwatcher.jobs):
+            has_history = self.urlwatcher.cache_storage.has_history_data(job.get_guid())
+            if not has_history:
+                logger.info('Add Job: %s', job.pretty_name())
+                new_jobs.append(idx+1)
+        if not new_jobs:
+            return 0
+        self.urlwatch_config.idx_set = frozenset(new_jobs)
+        self.urlwatcher.run_jobs()
+        self.urlwatcher.close()
+
     def _resolve_job_history(self, id, max_entries=10):
         job = self._get_job(id)
 
@@ -274,6 +287,8 @@ def handle_actions(self):
             sys.exit(self.test_filter(self.urlwatch_config.test_filter))
         if self.urlwatch_config.test_diff_filter:
             sys.exit(self.test_diff_filter(self.urlwatch_config.test_diff_filter))
+        if self.urlwatch_config.prepare_jobs:
+            sys.exit(self.prepare_jobs())
         if self.urlwatch_config.dump_history:
             sys.exit(self.dump_history(self.urlwatch_config.dump_history))
         if self.urlwatch_config.list:

lib/urlwatch/config.py

@@ -93,6 +93,7 @@ def parse_args(self, cmdline_args):
         group.add_argument('--delete', metavar='JOB', help='delete job by location or index')
         group.add_argument('--enable', metavar='JOB', help='enable job by location or index')
         group.add_argument('--disable', metavar='JOB', help='disable job by location or index')
+        group.add_argument('--prepare-jobs', action='store_true', help='run new jobs or jobs without history')
         group.add_argument('--change-location', metavar=('JOB', 'NEW_LOCATION'), nargs=2, help='change the location of an existing job by location or index')
         group.add_argument('--test-filter', metavar='JOB', help='test filter output of job by location or index')
         group.add_argument('--test-diff-filter', metavar='JOB',

lib/urlwatch/storage.py

@@ -567,6 +567,8 @@ def __init__(self, filename):
         self.db = minidb.Store(self.filename, debug=True, vacuum_on_close=False)
         self.db.register(CacheEntry)
 
+        self._cached_has_history_data_list = []
+
     def close(self):
         self.db.close()
         self.db = None
@@ -600,6 +602,18 @@ def get_history_data(self, guid, count=1):
                     break
         return history
 
+    def has_history_data(self, guid):
+        if not self._cached_has_history_data_list:
+            for data in list(CacheEntry.query(self.db, CacheEntry.c.guid,
+                                              where=((CacheEntry.c.tries == 0) | (CacheEntry.c.tries == None)))):  # noqa:E711
+                self._cached_has_history_data_list.append(data[0])
+        try:
+            if self._cached_has_history_data_list.index(guid):
+                return True
+        except ValueError:
+            return False
+        return False
+
     def save(self, job, guid, data, timestamp, tries, etag=None):
         self.db.save(CacheEntry(guid=guid, timestamp=timestamp, data=data, tries=tries, etag=etag))
         self.db.commit()