Merge pull request #55 from robgjansen/configurable-xz-threads

Configurable number of xz compression threads

tools/tgentools/analysis.py

@@ -87,7 +87,7 @@ def merge(self, analysis):
             else:
                 self.json_db['data'][nickname] = analysis.json_db['data'][nickname]
 
-    def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True):
+    def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True, xz_nthreads=cpu_count()):
         if filename is None:
             if self.date_filter is None:
                 filename = "tgen.analysis.json.xz"
@@ -100,7 +100,7 @@ def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True):
 
         logging.info("saving analysis results to {}".format(filepath))
 
-        outf = util.FileWritable(filepath, do_compress=do_compress)
+        outf = util.FileWritable(filepath, do_compress=do_compress, xz_nthreads=xz_nthreads)
         json.dump(self.json_db, outf, sort_keys=True, separators=(',', ': '), indent=2)
         outf.close()
 

tools/tgentools/tgentools

@@ -286,11 +286,11 @@ def analyze(args):
         analysis = ParallelAnalysis(nickname=args.nickname, ip_address=args.ip_address)
         analysis.analyze(paths, do_complete=args.do_complete, date_filter=args.date_filter,
             num_subprocs=min(args.nprocesses, len(paths)))
+        analysis.save(output_prefix=args.prefix, do_compress=True, xz_nthreads=args.nprocesses)
     else:
         analysis = SerialAnalysis(nickname=args.nickname, ip_address=args.ip_address)
         analysis.analyze(paths, do_complete=args.do_complete, date_filter=args.date_filter)
-
-    analysis.save(output_prefix=args.prefix)
+        analysis.save(output_prefix=args.prefix, do_compress=True, xz_nthreads=1)
 
 def visualize(args):
     from tgentools.visualization import TGenVisualization

tools/tgentools/util.py

@@ -202,10 +202,11 @@ def close(self):
 
 class FileWritable(Writable):
 
-    def __init__(self, filename, do_compress=False, do_truncate=False):
+    def __init__(self, filename, do_compress=False, do_truncate=False, xz_nthreads=3):
         self.filename = filename
         self.do_compress = do_compress
         self.do_truncate = do_truncate
+        self.xz_nthreads = xz_nthreads
         self.file = None
         self.xzproc = None
         self.ddproc = None
@@ -231,7 +232,7 @@ def open(self):
 
     def __open_nolock(self):
         if self.do_compress:
-            self.xzproc = Popen("xz --threads=3 -".split(), stdin=PIPE, stdout=PIPE)
+            self.xzproc = Popen(f"xz --threads={self.xz_nthreads} -".split(), stdin=PIPE, stdout=PIPE)
             dd_cmd = "dd of={0}".format(self.filename)
             # # note: its probably not a good idea to append to finalized compressed files
             # if not self.do_truncate: dd_cmd += " oflag=append conv=notrunc"