From 9fc8fefaf98defdbe1151ba6504f99421424c188 Mon Sep 17 00:00:00 2001 From: Louis-Francois Meunier Date: Thu, 12 Jan 2023 11:17:00 +0100 Subject: [PATCH] Fix code for tighter PEP8 checks Remove footprtins' historical style (it is not so bad after all). --- src/taylorism/__init__.py | 52 ++++++++--------- src/taylorism/examples.py | 110 ++++++++++++++++++------------------ src/taylorism/schedulers.py | 108 +++++++++++++++++------------------ tests/test_taylorism.py | 74 ++++++++++++++---------- 4 files changed, 178 insertions(+), 166 deletions(-) diff --git a/src/taylorism/__init__.py b/src/taylorism/__init__.py index 13b70769a..8a3e793e0 100644 --- a/src/taylorism/__init__.py +++ b/src/taylorism/__init__.py @@ -120,36 +120,36 @@ class Worker(FootprintBase): _abstract = True _collector = ('worker',) _footprint = dict( - attr = dict( - name = dict( - info = 'Name of the worker.', - optional = True, - default = None, - access = 'rwx', + attr=dict( + name=dict( + info='Name of the worker.', + optional=True, + default=None, + access='rwx', ), - memory = dict( - info = 'Memory that should be used by the worker (in MiB).', - optional = True, - default = 0., - type = float, + memory=dict( + info='Memory that should be used by the worker (in MiB).', + optional=True, + default=0., + type=float, ), - expected_time = dict( - info = 'How long the worker is expected to run (in s).', - optional = True, - default = 0., - type = float, + expected_time=dict( + info='How long the worker is expected to run (in s).', + optional=True, + default=0., + type=float, ), - scheduler_ticket = dict( - info = 'The slot number given by the scheduler (optional).', - optional = True, - default = None, - type = int, + scheduler_ticket=dict( + info='The slot number given by the scheduler (optional).', + optional=True, + default=None, + type=int, ), - scheduler_hooks = dict( - info = 'List of callbacks before starting effective task work.', - optional = True, - default = FPList(), - type = FPList, + scheduler_hooks=dict( + info='List of callbacks before starting effective task work.', + optional=True, + default=FPList(), + type=FPList, ), ) ) diff --git a/src/taylorism/examples.py b/src/taylorism/examples.py index b3bc8ba49..4b3bad8b9 100644 --- a/src/taylorism/examples.py +++ b/src/taylorism/examples.py @@ -24,18 +24,18 @@ class Sleeper(Worker): """ _footprint = dict( - info = "Sleeps.", - attr = dict( - sleeping_time = dict( - info = "Sleeping time in s.", - values = [0.001, 0.01, 0.1] + list(range(10)) + list(range(10, 65, 5)), - type = float, + info="Sleeps.", + attr=dict( + sleeping_time=dict( + info="Sleeping time in s.", + values=[0.001, 0.01, 0.1] + list(range(10)) + list(range(10, 65, 5)), + type=float, ), - wakeup_sentence = dict( - info = "What to say after sleep.", - optional = True, - access = 'rwx', - default = 'Hello !', + wakeup_sentence=dict( + info="What to say after sleep.", + optional=True, + access='rwx', + default='Hello !', ), ) ) @@ -58,14 +58,14 @@ class Logarithmer(Worker): an example of using shared numpy arrays among workers. """ _footprint = dict( - attr = dict( - row = dict( - info = "Index of the row of the array on which the Worker is supposed to work.", - type = int), - array = dict( - info = "The shared-memory array on which to work on.", - type = SharedNumpyArray, - access = 'rwx') + attr=dict( + row=dict( + info="Index of the row of the array on which the Worker is supposed to work.", + type=int), + array=dict( + info="The shared-memory array on which to work on.", + type=SharedNumpyArray, + access='rwx') ) ) @@ -82,17 +82,17 @@ class Summer(Worker): This needs and illustrates process-safetiness. """ _footprint = dict( - attr = dict( - value = dict( - info = "Value to be added by the worker.", - type = int), - shared_sum = dict( - info = "The shared-memory array on which to sum.", - type = SharedNumpyArray, - access = 'rwx'), - use_lock = dict( - info = "Whether to use the lock (thread-safe) or not (may lead to a wrong result !).", - type = bool) + attr=dict( + value=dict( + info="Value to be added by the worker.", + type=int), + shared_sum=dict( + info="The shared-memory array on which to sum.", + type=SharedNumpyArray, + access='rwx'), + use_lock=dict( + info="Whether to use the lock (thread-safe) or not (may lead to a wrong result !).", + type=bool) ) ) @@ -114,31 +114,31 @@ class MatrixProducter(Worker): Again, numpy matrix products may probably be more efficient... """ _footprint = dict( - attr = dict( - A = dict( - info = "The A shared-memory array matrix.", - type = SharedNumpyArray, - access = 'rwx'), - B = dict( - info = "The B shared-memory array matrix.", - type = SharedNumpyArray, - access = 'rwx'), - C = dict( - info = "The C shared-memory array matrix.", - type = SharedNumpyArray, - access = 'rwx'), - i1 = dict( - info = "The first index i of the the Worker is responsible for.", - type = int), - i2 = dict( - info = "The last index i of the the Worker is responsible for.", - type = int), - j1 = dict( - info = "The first index j of the the Worker is responsible for.", - type = int), - j2 = dict( - info = "The first index j of the the Worker is responsible for.", - type = int), + attr=dict( + A=dict( + info="The A shared-memory array matrix.", + type=SharedNumpyArray, + access='rwx'), + B=dict( + info="The B shared-memory array matrix.", + type=SharedNumpyArray, + access='rwx'), + C=dict( + info="The C shared-memory array matrix.", + type=SharedNumpyArray, + access='rwx'), + i1=dict( + info="The first index i of the the Worker is responsible for.", + type=int), + i2=dict( + info="The last index i of the the Worker is responsible for.", + type=int), + j1=dict( + info="The first index j of the the Worker is responsible for.", + type=int), + j2=dict( + info="The first index j of the the Worker is responsible for.", + type=int), ) ) diff --git a/src/taylorism/schedulers.py b/src/taylorism/schedulers.py index 38fac4631..da4f89571 100644 --- a/src/taylorism/schedulers.py +++ b/src/taylorism/schedulers.py @@ -54,11 +54,11 @@ class BaseScheduler(FootprintBase): _abstract = True _collector = ('scheduler',) _footprint = dict( - attr = dict( - identity = dict( - info = "Scheduler identity.", - optional = True, - default = 'anonymous', + attr=dict( + identity=dict( + info="Scheduler identity.", + optional=True, + default='anonymous', ), ) ) @@ -101,11 +101,11 @@ class NewLaxistScheduler(BaseScheduler): """No sorting is done !""" _footprint = dict( - attr = dict( - nosort = dict( - alias = ('laxist', 'unsorted'), - values = (True,), - type = bool, + attr=dict( + nosort=dict( + alias=('laxist', 'unsorted'), + values=(True,), + type=bool, ), ) ) @@ -124,10 +124,10 @@ class NewLimitedScheduler(BaseScheduler): _abstract = True, _footprint = dict( - attr = dict( - limit = dict( - values = ['threads', 'memory', 'mem', 'threads+memory'], - remap = dict(mem = 'memory'), + attr=dict( + limit=dict( + values=['threads', 'memory', 'mem', 'threads+memory'], + remap=dict(mem='memory'), ), ) ) @@ -179,15 +179,15 @@ class NewMaxThreadsScheduler(NewLimitedScheduler): _footprint = [ _binded_fpattr, dict( - attr = dict( - limit = dict( - values = ['threads', 'processes'], - remap = dict(processes = 'threads'), + attr=dict( + limit=dict( + values=['threads', 'processes'], + remap=dict(processes='threads'), ), - max_threads = dict( - alias = ('maxpc', 'maxthreads'), - remap = {0: multiprocessing.cpu_count() / 2}, - type = int, + max_threads=dict( + alias=('maxpc', 'maxthreads'), + remap={0: multiprocessing.cpu_count() / 2}, + type=int, ), ) ) @@ -217,30 +217,30 @@ class NewMaxMemoryScheduler(NewLimitedScheduler): _footprint = [ _binded_fpattr, dict( - attr = dict( - limit = dict( - values = ['memory', 'mem'], - remap = dict(mem = 'memory'), + attr=dict( + limit=dict( + values=['memory', 'mem'], + remap=dict(mem='memory'), ), - max_memory = dict( - info = "Amount of usable memroy (in MiB)", - optional = True, - type = float, - access = 'rwx', + max_memory=dict( + info="Amount of usable memroy (in MiB)", + optional=True, + type=float, + access='rwx', ), - memory_per_task = dict( - info = ("If a worker do not provide any information on memory, " + - "request at least *memory_per_task* MiB of memory."), - optional = True, - default = 2048., - type = float, + memory_per_task=dict( + info=("If a worker do not provide any information on memory, " + + "request at least *memory_per_task* MiB of memory."), + optional=True, + default=2048., + type=float, ), - memory_max_percentage = dict( - info = ("Max memory level as a fraction of the total" + - "system memory (used only if max_memroy is not provided)."), - optional = True, - default = 0.75, - type = float, + memory_max_percentage=dict( + info=("Max memory level as a fraction of the total" + + "system memory (used only if max_memroy is not provided)."), + optional=True, + default=0.75, + type=float, ), ) ) @@ -293,14 +293,14 @@ class LongerFirstScheduler(NewMaxMemoryScheduler): """ _footprint = dict( - attr = dict( - limit = dict( - values = ['threads+memory'], + attr=dict( + limit=dict( + values=['threads+memory'], ), - max_threads = dict( - alias = ('maxpc', 'maxthreads'), - remap = {0: multiprocessing.cpu_count() / 2}, - type = int, + max_threads=dict( + alias=('maxpc', 'maxthreads'), + remap={0: multiprocessing.cpu_count() / 2}, + type=int, ), ) ) @@ -338,10 +338,10 @@ class NewSingleOpenFileScheduler(NewMaxThreadsScheduler): """ _footprint = dict( - attr = dict( - singlefile = dict( - values = (True,), - type = bool, + attr=dict( + singlefile=dict( + values=(True,), + type=bool, ), ) ) diff --git a/tests/test_taylorism.py b/tests/test_taylorism.py index a76fad599..43b25fe59 100644 --- a/tests/test_taylorism.py +++ b/tests/test_taylorism.py @@ -4,11 +4,6 @@ import sys import time -numpy_looks_fine = True -try: - import numpy as np -except ImportError: - numpy_looks_fine = False from unittest import TestCase, main, skipIf @@ -19,6 +14,13 @@ from bronx.system import interrupt # because subprocesses must be killable properly from bronx.system import cpus as cpus_tool +try: + import numpy as np +except ImportError: + numpy_looks_fine = False +else: + numpy_looks_fine = True + tloglevel = 'CRITICAL' tloglevel_taylorism = 'CRITICAL' @@ -43,21 +45,21 @@ class Succeeder(examples.Sleeper): """Does nothing, but succeeds at it.""" _footprint = dict( - priority = dict( - level = footprints.priorities.top.level('debug') + priority=dict( + level=footprints.priorities.top.level('debug') ), - info = "Suceeds.", - attr = dict( - succeed = dict( - info = "Supposed to succeed.", - type = bool, - values = [True] + info="Suceeds.", + attr=dict( + succeed=dict( + info="Supposed to succeed.", + type=bool, + values=[True] ), - bind_test = dict( - info = "Do the bind test.", - type = bool, - optional = True, - default = False, + bind_test=dict( + info="Do the bind test.", + type=bool, + optional=True, + default=False, ) ) ) @@ -75,15 +77,15 @@ class Failer(examples.Sleeper): """Does nothing, but fails at it.""" _footprint = dict( - priority = dict( - level = footprints.priorities.top.level('debug') + priority=dict( + level=footprints.priorities.top.level('debug') ), - info = "Fails.", - attr = dict( - succeed = dict( - info = "Supposed to fail.", - type = bool, - values = [False] + info="Fails.", + attr=dict( + succeed=dict( + info="Supposed to fail.", + type=bool, + values=[False] ), ) ) @@ -143,10 +145,13 @@ def test_servermode(self): scheduler=scheduler, ) time.sleep(0.2) - boss.set_instructions(dict(succeed=True,), individual_instructions=dict(sleeping_time=[0.001, ])) + boss.set_instructions(dict(succeed=True,), + individual_instructions=dict(sleeping_time=[0.001, ])) boss.wait_till_finished() report = boss.get_report() - self.assertEqual(len(report['workers_report']), 4, "4 instructions have been sent, which is not the size of report.") + self.assertEqual(len(report['workers_report']), + 4, + "4 instructions have been sent, which is not the size of report.") @stderr2out_deco def test_toomany_instr_after_crash(self): @@ -200,7 +205,9 @@ def test_redundant_workers_name(self): with self.assertRaises(ValueError): boss = taylorism.run_as_server( common_instructions=dict(), - individual_instructions=dict(name=['alfred', 'alfred'], sleeping_time=[60, 60], succeed=[True, True]), + individual_instructions=dict(name=['alfred', 'alfred'], + sleeping_time=[60, 60], + succeed=[True, True]), scheduler=footprints.proxy.scheduler(limit='threads', max_threads=2), ) boss.wait_till_finished() @@ -215,7 +222,9 @@ def test_expansion_workers_name(self): ) boss.wait_till_finished() report = boss.get_report() - self.assertEqual(len(report['workers_report']), 2, "2 instructions have been sent, which is not the size of report.") + self.assertEqual(len(report['workers_report']), + 2, + "2 instructions have been sent, which is not the size of report.") @skipIf(not numpy_looks_fine, "NumPy is unavailable.") def test_sharedmemory_array(self): @@ -230,7 +239,10 @@ def test_sharedmemory_array(self): sharedmemory_common_instructions=dict(shared_sum=s) ) boss.wait_till_finished() - self.assertEqual(s[0], sum(vals), "sharedmemory array has wrong value:{} instead of expected: {}.".format(s[0], sum(vals))) + self.assertEqual(s[0], + sum(vals), + "sharedmemory array has wrong value:{} instead of expected: {}." + .format(s[0], sum(vals))) if __name__ == '__main__':