lowering memory requests, adding exponential mem increase on batch jo…

…b retry (#73)

* adding exponential mem increase on batch job retry
* reducing mem reqs for meta-zoom

batch-setup/make_meta_tiles.py

@@ -291,7 +291,7 @@ def _big_jobs(rawr_bucket, prefix, key_format_type, rawr_zoom, group_zoom,
     return big_jobs
 
 
-def enqueue_tiles(config_file, tile_list_file, check_metatile_exists):
+def enqueue_tiles(config_file, tile_list_file, check_metatile_exists, mem_multiplier=1.0, mem_max=32 * 1024):
     from tilequeue.command import make_config_from_argparse
     from tilequeue.command import tilequeue_batch_enqueue
     from make_rawr_tiles import BatchEnqueueArgs
@@ -301,9 +301,15 @@ def enqueue_tiles(config_file, tile_list_file, check_metatile_exists):
         str(check_metatile_exists).lower())
     with open(args.config) as fh:
         cfg = make_config_from_argparse(fh)
+
+    update_memory_request(cfg, mem_multiplier, mem_max)
     tilequeue_batch_enqueue(cfg, args)
 
 
+def update_memory_request(cfg, mem_multiplier, mem_max):
+    cfg.yml["batch"]["memory"] = int(min(cfg.yml["batch"]["memory"] * mem_multiplier, mem_max))
+
+
 # adaptor class for MissingTiles to see just the high zoom parts, this is used
 # along with the LowZoomLense to loop over missing tiles generically but
 # separately.
@@ -342,7 +348,10 @@ def _missing(self):
             self.split_zoom, self.zoom_max, self.big_jobs)
 
     def render(self, num_retries, lense):
+        mem_max = 32 * 1024  # 32 GiB
+
         for retry_number in range(0, num_retries):
+            mem_multiplier = 1.5 ** retry_number
             with self._missing() as missing:
                 missing_tile_file = lense.missing_file(missing)
                 count = wc_line(missing_tile_file)
@@ -361,8 +370,9 @@ def render(self, num_retries, lense):
                     sample = head_lines(missing_tile_file, 10)
                     print("Enqueueing %d %s tiles (e.g. %s)" %
                           (count, lense.description, ', '.join(sample)))
+
                     enqueue_tiles(lense.config, missing_tile_file,
-                                  check_metatile_exists)
+                                  check_metatile_exists, mem_multiplier, mem_max)
 
         else:
             with self._missing() as missing:

batch-setup/make_tiles.py

@@ -134,7 +134,7 @@ def _chr_range(a, b):
 # raised if jobs fail with out-of-memory errors.
 memory = {
     'rawr-batch': 8192,
-    'meta-batch': 8192,  # 8 GiB
+    'meta-batch': 4096,  # 4 GiB
     'meta-low-zoom-batch': 8192,  # 8 GiB
     'missing-meta-tiles-write': 1024,
 }