diff --git a/.github/workflows/conda_env/environment_static_analysis.yml b/.github/workflows/conda_env/environment_static_analysis.yml index 25c9e6cd..cfc72bea 100644 --- a/.github/workflows/conda_env/environment_static_analysis.yml +++ b/.github/workflows/conda_env/environment_static_analysis.yml @@ -2,8 +2,8 @@ name: ips_static_analysis channels: - conda-forge dependencies: -- python=3.9 -- ruff=0.9.4 +- python=3.10 +- ruff=0.16.0 #- flake8=5.0.4 #- pylint=2.15.3 #- bandit=1.7.4 diff --git a/.gitignore b/.gitignore index fd04f644..ea908a6b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ www/ log resource_usage work/ +ENSEMBLES # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/components/drivers/hello/hello_worker_task_pool.py b/components/drivers/hello/hello_worker_task_pool.py index f7ba13b0..ed8a9274 100755 --- a/components/drivers/hello/hello_worker_task_pool.py +++ b/components/drivers/hello/hello_worker_task_pool.py @@ -8,10 +8,10 @@ from time import asctime, sleep -def myFun(*args): - print(f"{asctime()} : Running myFUN {args}") +def my_fun(*args): + print(f"{asctime()} : Running my_fun {args}") sleep(int(args[0])) - print(f"{asctime()} : Finished myFUN {args}") + print(f"{asctime()} : Finished my_fun {args}") return 0 @@ -46,10 +46,10 @@ def step(self, timeStamp=0.0): logfile=f"task_{i}.log", task_env=task_env) self.services.add_task('pool', 'method_'+str(i), 1, - cwd, copy.copy(self).myMethod, str(duration[i]), + cwd, copy.copy(self).my_method, str(duration[i]), task_env=task_env) self.services.add_task('pool', 'function_' + str(i), 1, - cwd, myFun, str(duration[i]), + cwd, my_fun, str(duration[i]), task_env=task_env) ret_val = self.services.submit_tasks('pool', use_dask=True, dask_nodes=1, dask_ppw=10) @@ -79,10 +79,10 @@ def step(self, timeStamp=0.0): return - def myMethod(self, *args): - print(f"{asctime()} : Running myMethod {args} self.BIN_PATH = {self.BIN_PATH}") + def my_method(self, *args): + print(f"{asctime()} : Running my_method {args} self.BIN_PATH = {self.BIN_PATH}") sleep(int(args[0])) - print(f"{asctime()} : Finished myMethod {args} self.BIN_PATH = {self.BIN_PATH}") + print(f"{asctime()} : Finished my_method {args} self.BIN_PATH = {self.BIN_PATH}") return 0 def finalize(self, timeStamp=0.0): diff --git a/doc/development.rst b/doc/development.rst index c989c203..a79c310d 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -104,29 +104,29 @@ and the output will look like ipsframework/__init__.py 11 0 100% ipsframework/cca_es_spec.py 62 10 84% ipsframework/component.py 105 19 82% - ipsframework/componentRegistry.py 105 25 76% - ipsframework/configurationManager.py 510 103 80% + ipsframework/component_registry.py 105 25 76% + ipsframework/configuration_manager.py 510 103 80% ipsframework/convert_log_function.py 29 1 97% - ipsframework/dataManager.py 72 15 79% + ipsframework/data_manager.py 72 15 79% ipsframework/debug.py 3 0 100% - ipsframework/eventService.py 137 53 61% - ipsframework/eventServiceProxy.py 118 49 58% + ipsframework/event_service.py 137 53 61% + ipsframework/event_service_proxy.py 118 49 58% ipsframework/ips.py 360 51 86% - ipsframework/ipsExceptions.py 61 2 97% - ipsframework/ipsLogging.py 92 8 91% + ipsframework/ips_exceptions.py 61 2 97% + ipsframework/ips_logging.py 92 8 91% ipsframework/ips_es_spec.py 43 7 84% ipsframework/ipsutil.py 73 26 64% ipsframework/messages.py 58 0 100% ipsframework/node_structure.py 193 31 84% ipsframework/platformspec.py 18 4 78% - ipsframework/portalBridge.py 205 36 82% - ipsframework/resourceHelper.py 304 59 81% - ipsframework/resourceManager.py 340 69 80% - ipsframework/runspaceInitComponent.py 88 31 65% - ipsframework/sendPost.py 41 2 95% + ipsframework/portal_bridge.py 205 36 82% + ipsframework/resource_helper.py 304 59 81% + ipsframework/resource_manager.py 340 69 80% + ipsframework/runspace_init_component.py 88 31 65% + ipsframework/send_post.py 41 2 95% ipsframework/services.py 1200 234 80% - ipsframework/taskManager.py 322 74 77% - ipsframework/topicManager.py 59 5 92% + ipsframework/task_manager.py 322 74 77% + ipsframework/topic_manager.py 59 5 92% ----------------------------------------------------------- TOTAL 4609 914 80% diff --git a/doc/examples/dask/dask_worker.py b/doc/examples/dask/dask_worker.py index 3ff97ad6..41e63a36 100644 --- a/doc/examples/dask/dask_worker.py +++ b/doc/examples/dask/dask_worker.py @@ -3,8 +3,8 @@ from ipsframework import Component -def myFun(*args): - print(f"myFun({args[0]})") +def my_fun(*args): + print(f"my_fun({args[0]})") sleep(float(args[0])) return 0 @@ -16,8 +16,8 @@ def step(self, timestamp=0.0): duration = 0.5 self.services.add_task('pool', 'binary', 1, cwd, self.EXECUTABLE, duration) - self.services.add_task('pool', 'function', 1, cwd, myFun, duration) - self.services.add_task('pool', 'method', 1, cwd, copy.copy(self).myMethod, duration) + self.services.add_task('pool', 'function', 1, cwd, my_fun, duration) + self.services.add_task('pool', 'method', 1, cwd, copy.copy(self).my_method, duration) ret_val = self.services.submit_tasks('pool', use_dask=True, @@ -26,7 +26,7 @@ def step(self, timestamp=0.0): exit_status = self.services.get_finished_tasks('pool') print('exit_status = ', exit_status) - def myMethod(self, *args): - print(f"myMethod({args[0]})") + def my_method(self, *args): + print(f"my_method({args[0]})") sleep(float(args[0])) return 0 diff --git a/doc/examples/dask/simulation_log.json b/doc/examples/dask/simulation_log.json index 1d7b88fd..07d66c32 100644 --- a/doc/examples/dask/simulation_log.json +++ b/doc/examples/dask/simulation_log.json @@ -8,7 +8,7 @@ "code": "DASK_WORKER__DaskWorker", "eventtype": "IPS_LAUNCH_DASK_TASK", "walltime": "2.33", - "comment": "task_name = function, Target = myFun(0.5)", + "comment": "task_name = function, Target = my_fun(0.5)", } { "code": "DASK_WORKER__DaskWorker", diff --git a/doc/examples/ensembles/a_comp.py b/doc/examples/ensembles/a_comp.py index 5f40b066..e6964101 100644 --- a/doc/examples/ensembles/a_comp.py +++ b/doc/examples/ensembles/a_comp.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python3 """ Component wrapper for the ensemble example for `a_sim`. """ from ipsframework import Component -from ipsframework.resourceHelper import get_platform_info +from ipsframework.resource_helper import get_platform_info -class a_comp(Component): +class AComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) @@ -17,4 +16,3 @@ def step(self, timestamp=0.0): run_env = get_platform_info() self.services.info(run_env) - diff --git a/doc/examples/ensembles/another_comp.py b/doc/examples/ensembles/another_comp.py index d0aeb97e..6707d79a 100644 --- a/doc/examples/ensembles/another_comp.py +++ b/doc/examples/ensembles/another_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """ Component wrapper for the ensemble example for `another_sim`. """ from ipsframework import Component -from ipsframework.resourceHelper import get_platform_info +from ipsframework.resource_helper import get_platform_info -class another_comp(Component): +class AnotherComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) @@ -17,4 +16,4 @@ def step(self, timestamp=0.0): print(f'another_comp parameters: D={self.D}, B={self.B}, F={self.F}') run_env = get_platform_info() - self.services.info(run_env) \ No newline at end of file + self.services.info(run_env) diff --git a/doc/examples/ensembles/driver.config b/doc/examples/ensembles/driver.config index eeb070e7..49bff3da 100644 --- a/doc/examples/ensembles/driver.config +++ b/doc/examples/ensembles/driver.config @@ -17,7 +17,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/ CLASS = driver SUB_CLASS = - NAME = ensemble_driver + NAME = EnsembleDriver SCRIPT = ${BIN_PATH}/driver.py NPROC = 1 # Ensures we can find this file for run_ensembles call. @@ -26,5 +26,3 @@ SIMULATION_MODE = NORMAL INPUT_FILES = template.config OUTPUT_FILES = RESTART_FILES = - - diff --git a/doc/examples/ensembles/driver.py b/doc/examples/ensembles/driver.py index 3155a747..907f1f07 100644 --- a/doc/examples/ensembles/driver.py +++ b/doc/examples/ensembles/driver.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python3 """ - Example driver for the ensembles example. + Example Driver for the ensembles example. Please note that run_ensemble can be run from any IPS component, not - just a driver. The driver is used here for simplicity. + just a Driver. The Driver is used here for simplicity. """ import os from pathlib import Path @@ -11,11 +10,11 @@ from ipsframework import Component -class ensemble_driver(Component): +class EnsembleDriver(Component): def __init__(self, services, config): super().__init__(services, config) - print('Creating driver') + print('Creating Driver') def init(self, timeStamp=0.0): return @@ -51,7 +50,7 @@ def step(self, timestamp=0.0, **keywords): # variable values. `mapping` is a data struct that associates the # specific simulation to a given run directory so that the user can # easily find output for a specific run. Note that "template.config" - # is in INPUT_FILE for the driver component so that it is copied to the run. + # is in INPUT_FILE for the Driver component so that it is copied to the run. # But, of course, you could also use a full path to the file, instead. mapping = self.services.run_ensemble('template.config', variables, '/tmp/IPS', name="EXAMPLE_", @@ -62,4 +61,3 @@ def step(self, timestamp=0.0, **keywords): def finalize(self, timeStamp=0.0): return - diff --git a/doc/examples/ensembles/environment.py b/doc/examples/ensembles/environment.py index 7ffc24fe..d929ec2f 100644 --- a/doc/examples/ensembles/environment.py +++ b/doc/examples/ensembles/environment.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Provides information about the runtime environment """ diff --git a/doc/examples/ensembles/instance_driver.py b/doc/examples/ensembles/instance_driver.py index 3fbd9fb9..4f67c2e7 100644 --- a/doc/examples/ensembles/instance_driver.py +++ b/doc/examples/ensembles/instance_driver.py @@ -1,18 +1,17 @@ -#!/usr/bin/env python3 """ - Example driver for the ensembles example. + Example Driver for the ensembles example. Please note that run_ensemble can be run from any IPS component, not - just a driver. The driver is used here for simplicity. + just a Driver. The Driver is used here for simplicity. """ from ipsframework import Component -class instance_driver(Component): +class InstanceDriver(Component): def __init__(self, services, config): super().__init__(services, config) - print('Creating instance driver') + print('Creating instance Driver') def init(self, timeStamp=0.0): return @@ -22,7 +21,7 @@ def step(self, timestamp=0.0, **keywords): In this example we have two components for an example coupled simulation. The components are - 'a_sim_comp' and 'another_sim_comp'. Here, we step + 'ASimComp' and 'AnotherSimComp'. Here, we step each of those components where they echo their unique parameters. """ @@ -35,4 +34,3 @@ def step(self, timestamp=0.0, **keywords): self.services.call(another_comp, 'step', 0.0) self.services.info('Finished stepping components') - diff --git a/doc/examples/ensembles/template.config b/doc/examples/ensembles/template.config index 47678bfc..94a88132 100644 --- a/doc/examples/ensembles/template.config +++ b/doc/examples/ensembles/template.config @@ -24,7 +24,7 @@ SRC_DIR = /Users/may/Projects/IPS BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/ CLASS = driver SUB_CLASS = - NAME = instance_driver + NAME = InstanceDriver SCRIPT = ${BIN_PATH}/instance_driver.py NPROC = 1 INPUT_FILES = @@ -36,7 +36,7 @@ SRC_DIR = /Users/may/Projects/IPS BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/ CLASS = workers SUB_CLASS = - NAME = a_comp + NAME = AComp SCRIPT = ${BIN_PATH}/a_comp.py NPROC = 1 INPUT_FILES = @@ -51,7 +51,7 @@ SRC_DIR = /Users/may/Projects/IPS BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/ CLASS = workers SUB_CLASS = - NAME = another_comp + NAME = AnotherComp SCRIPT = ${BIN_PATH}/another_comp.py NPROC = 1 INPUT_FILES = diff --git a/doc/examples/helloworld.config b/doc/examples/helloworld.config index d7a1b819..e116382a 100644 --- a/doc/examples/helloworld.config +++ b/doc/examples/helloworld.config @@ -15,7 +15,7 @@ SIMULATION_MODE = NORMAL [hello_world_driver] CLASS = driver SUB_CLASS = - NAME = hello_driver + NAME = HelloDriver NPROC = 1 BIN_PATH = INPUT_FILES = @@ -26,7 +26,7 @@ SIMULATION_MODE = NORMAL [hello_world] CLASS = workers SUB_CLASS = - NAME = hello_worker + NAME = HelloWorker NPROC = 1 BIN_PATH = INPUT_FILES = diff --git a/doc/examples/helloworld/helloworld/hello_driver.py b/doc/examples/helloworld/helloworld/hello_driver.py index 2d28b73d..096f5d5b 100644 --- a/doc/examples/helloworld/helloworld/hello_driver.py +++ b/doc/examples/helloworld/helloworld/hello_driver.py @@ -1,13 +1,13 @@ from ipsframework import Component -class hello_driver(Component): +class HelloDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) def step(self, timestamp=0.0): - print('hello_driver: beginning step call') + print('HelloDriver: beginning step call') worker_comp = self.services.get_port('WORKER') self.services.call(worker_comp, 'step', 0.0) - print('hello_driver: finished step call') + print('HelloDriver: finished step call') diff --git a/doc/examples/helloworld/helloworld/hello_worker.py b/doc/examples/helloworld/helloworld/hello_worker.py index 5fb97d9f..a79ac202 100644 --- a/doc/examples/helloworld/helloworld/hello_worker.py +++ b/doc/examples/helloworld/helloworld/hello_worker.py @@ -1,10 +1,10 @@ from ipsframework import Component -class hello_worker(Component): +class HelloWorker(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) def step(self, timestamp=0.0): - print('Hello from hello_worker') + print('Hello from HelloWorker') diff --git a/doc/examples/helloworld/setup.py b/doc/examples/helloworld/setup.py index f53bcb1a..dd618e7b 100644 --- a/doc/examples/helloworld/setup.py +++ b/doc/examples/helloworld/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 from setuptools import setup, find_packages setup( diff --git a/doc/the_code.rst b/doc/the_code.rst index 3e8b9562..ae0450f3 100644 --- a/doc/the_code.rst +++ b/doc/the_code.rst @@ -13,19 +13,19 @@ Framework Data Manager ------------ -.. automodule:: ipsframework.dataManager +.. automodule:: ipsframework.data_manager :members: :undoc-members: Task Manager ------------ -.. automodule:: ipsframework.taskManager +.. automodule:: ipsframework.task_manager :members: :undoc-members: Resource Manager ---------------- -.. automodule:: ipsframework.resourceManager +.. automodule:: ipsframework.resource_manager :members: :undoc-members: @@ -45,7 +45,7 @@ Resource Manager ---------------------------------- -.. automodule:: ipsframework.resourceHelper +.. automodule:: ipsframework.resource_helper :members: :undoc-members: @@ -58,13 +58,13 @@ Component Component Registry ------------------ -.. automodule:: ipsframework.componentRegistry +.. automodule:: ipsframework.component_registry :members: :undoc-members: Configuration Manager --------------------- -.. automodule:: ipsframework.configurationManager +.. automodule:: ipsframework.configuration_manager :members: :undoc-members: @@ -81,7 +81,7 @@ Other Utilities IPS Exceptions ^^^^^^^^^^^^^^ -.. automodule:: ipsframework.ipsExceptions +.. automodule:: ipsframework.ips_exceptions :members: :undoc-members: @@ -105,7 +105,6 @@ Framework Components ---------------------------------- -.. automodule:: ipsframework.runspaceInitComponent +.. automodule:: ipsframework.runspace_init_component :members: :undoc-members: - diff --git a/doc/user_guides/advanced_guide.rst b/doc/user_guides/advanced_guide.rst index c1ece0a3..9f81070a 100644 --- a/doc/user_guides/advanced_guide.rst +++ b/doc/user_guides/advanced_guide.rst @@ -113,14 +113,14 @@ As you can see in the example component, almost everything is specified in the c Drivers access components by their port names (as specified in the configuration file). To add a new component to the driver you will either need to add a new port name or use an existing port name. ``ips/components/drivers/dbb/generic_driver.py`` is a good all-purpose driver that most components should be able to use. If you are using an existing port name, then the code should just work. It is recommended to go through the driver code to make sure the component is being used in the expected manner. To add a new port name, you will need to add code to *generic_driver.step()*: * get a reference to the port (*self.services.get_port()*) -* call "init" on that component (*self.services.call(comp_ref, "init")*) +* call "init" on that component (*self.services.call(comp_ref, "init")*) * call "step" on that component (*self.services.call(comp_ref, "step")*) * call "finalize" on that component (*self.services.call(comp_ref, "finalize")*) The following sections of the configuration file may need to be modified. If you are not adding the component to an existing simulation, you can copy a configuration file from the examples directory and modify it. 1. *Plasma State (Shared Files) Section* - + You will need to modify this section to include any additional files needed by your component:: # Where to put plasma state files as the simulation evolves @@ -146,9 +146,9 @@ The following sections of the configuration file may need to be modified. If yo [PORTS] NAMES = INIT DRIVER MONITOR EPA NB - [[DRIVER]] + [[DRIVER]] IMPLEMENTATION = EPA_IC_FP_NB_DRIVER - [[INIT]] + [[INIT]] IMPLEMENTATION = minimal_state_init [[RF_IC]] IMPLEMENTATION = model_RF_IC @@ -178,8 +178,8 @@ The following sections of the configuration file may need to be modified. If yo # Time loop specification (two modes for now) EXPLICIT | REGULAR # For MODE = REGULAR, the framework uses the variables START, FINISH, and NSTEP - # For MODE = EXPLICIT, the framework uses the variable VALUES (space separated - # list of time values) + # For MODE = EXPLICIT, the framework uses the variable VALUES (space separated + # list of time values) [TIME_LOOP] MODE = EXPLICIT VALUES = 75.000 75.025 75.050 75.075 75.100 75.125 @@ -205,7 +205,7 @@ This section contains some useful tips on testing, debugging and documenting you * If this is a time stepping simulation, a small number of steps is useful because it will lead to shorter running times, allowing you to submit the job to a debug or other faster turnaround queue. * Debugging: - + * Add logging messages (*services.info()*, *services.warning()*, etc.) to make sure your component does what you think it does. * Remove other components from the simulation to figure out which one or which interaction is causing the problem * Take many checkpoints around the problem to narrow in on the problem. @@ -245,19 +245,19 @@ The framework will invoke the methods of the *INIT* and *DRIVER* components over * ``init_comp.finalize()`` - cleanup and confirmation of initialization * ``driver.init()`` - any initialization work (typically empty) * ``driver.step()`` - the bulk of the simulation - + * get references to the ports * call *init* on each port * get the time loop * implement logic of time stepping * during each time step: - * perform pre-step logic that may stage data or determine which components need to run or what parameters are given to each component + * perform pre-step logic that may stage data or determine which components need to run or what parameters are given to each component * call *step* on each port (as appropriate) * manage global plasma state at the end of each step * checkpoint components (frequency of checkpoints is controlled by framework) - * call *finalize* on each component + * call *finalize* on each component * ``driver.finalize()`` - any clean up activities (typically empty) @@ -282,7 +282,7 @@ The IPS framework contains a set of managers that perform services for the compo Component Invocation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Component invocation in the IPS means one component is calling another component's function. This API provides a mechanism to invoke methods on components through the framework. There are blocking and non-blocking versions, where the non-blocking versions require a second function to check the status of the call. Note that the *wait_call* has an optional argument (*block*) that changes when and what it returns. +Component invocation in the IPS means one component is calling another component's function. This API provides a mechanism to invoke methods on components through the framework. There are blocking and non-blocking versions, where the non-blocking versions require a second function to check the status of the call. Note that the *wait_call* has an optional argument (*block*) that changes when and what it returns. .. automethod:: ipsframework.services.ServicesProxy.call :noindex: @@ -524,7 +524,7 @@ If you try to launch a task with too many GPUs per node, *e.g.*: self.services.launch_task(8, cwd, "gpu-per-task", task_gpp=1) -then it will raise an :class:`~ipsframework.ipsExceptions.GPUResourceRequestMismatchException`. +then it will raise an :class:`~ipsframework.ips_exceptions.GPUResourceRequestMismatchException`. .. automethod:: ipsframework.services.ServicesProxy.launch_task :noindex: @@ -632,7 +632,7 @@ Logging The following logging methods can be used to write logging messages to the simulation log file. It is *strongly* recommended that these methods are used as opposed to print statements. The logging capability adds a timestamp and identifies the component that generated the message. The syntax for logging is a simple string or formatted string:: self.services.info('beginning step') - self.services.warning('unable to open log file %s for task %d, will use stdout instead', + self.services.warning('unable to open log file %s for task %d, will use stdout instead', logfile, task_id) There is no need to include information about the component in the message as the IPS logging interface includes a time stamp and information about what component sent the message:: diff --git a/doc/user_guides/component_package.rst b/doc/user_guides/component_package.rst index 33027fa8..8c8e260b 100644 --- a/doc/user_guides/component_package.rst +++ b/doc/user_guides/component_package.rst @@ -60,8 +60,8 @@ Or to install it in editable mode with With the components installed as a package you can reference them by ``MODULE`` instead of providing the full path with ``SCRIPT``. So to use -the `hello_driver` you do ``MODULE = helloworld.hello_driver``, and -for `hello_worker` you can do ``MODULE = helloworld.hello_worker``. +the `hello_driver` you do ``MODULE = helloworld.HelloDriver``, and +for `hello_worker` you can do ``MODULE = helloworld.HelloWorker``. A simple config to run this is, :download:`helloworld.config <../examples/helloworld.config>` diff --git a/doc/user_guides/dask.rst b/doc/user_guides/dask.rst index 7179c750..da4d2774 100644 --- a/doc/user_guides/dask.rst +++ b/doc/user_guides/dask.rst @@ -14,7 +14,7 @@ binary executables, you can run python functions and class methods. An example showing this is the following, where we are adding an executable (in this case :download:`sleep <../examples/dask/sleep>`), -a function that sleeps (``myFun``) and a method that sleeps +a function that sleeps (``my_fun``) and a method that sleeps (``myMethod``) respectively to a task pool and submitting the task pool with ``self.services.submit_tasks('pool', use_dask=True)``. @@ -41,7 +41,7 @@ executing: ... ret_val = 3 - myFun(0.5) + my_fun(0.5) myMethod(0.5) /bin/sleep 0.5 exit_status = {'binary': 0, 'method': 0, 'function': 0} diff --git a/doc/user_guides/ensembles.rst b/doc/user_guides/ensembles.rst index c4bc9c5c..33daed8d 100644 --- a/doc/user_guides/ensembles.rst +++ b/doc/user_guides/ensembles.rst @@ -322,8 +322,8 @@ The ensemble execution creates something like the following directory structure: │ │ ├── transport_comp.py │ │ └── instance_driver.py │ └── work - │ ├── driver__instance_driver_1 - │ ├── FWK_COMP_runspaceInitComponent_4 + │ ├── driver__InstanceDriver_1 + │ ├── FWK_COMP_RunspaceInitComponent_4 │ ├── workers__physics_comp_2 │ │ └── output.csv │ └── workers__transport_comp_3 diff --git a/doc/user_guides/migration.rst b/doc/user_guides/migration.rst index bfac1874..19bb07a0 100644 --- a/doc/user_guides/migration.rst +++ b/doc/user_guides/migration.rst @@ -49,7 +49,7 @@ These API have been deprecated for a long time and have been removed, you should +--------------------------------------------------------------------+----------------------------------+-----------------------------------------------------------------------------+ | class | removed API | new API | +====================================================================+==================================+=============================================================================+ -|:py:class:`~ipsframework.configurationManager.ConfigurationManager` | ``getPort()`` | :py:meth:`~ipsframework.configurationManager.ConfigurationManager.get_port` | +|:py:class:`~ipsframework.configuration_manager.ConfigurationManager` | ``getPort()`` | :py:meth:`~ipsframework.configuration_manager.ConfigurationManager.get_port` | +--------------------------------------------------------------------+----------------------------------+-----------------------------------------------------------------------------+ |:py:class:`~ipsframework.services.ServicesProxy` | ``getGlobalConfigParameter()`` | :py:meth:`~ipsframework.services.ServicesProxy.get_config_param` | +--------------------------------------------------------------------+----------------------------------+-----------------------------------------------------------------------------+ diff --git a/examples-proposed/001-helloworld/helloworld.conf b/examples-proposed/001-helloworld/helloworld.conf index d7a1b819..e116382a 100644 --- a/examples-proposed/001-helloworld/helloworld.conf +++ b/examples-proposed/001-helloworld/helloworld.conf @@ -15,7 +15,7 @@ SIMULATION_MODE = NORMAL [hello_world_driver] CLASS = driver SUB_CLASS = - NAME = hello_driver + NAME = HelloDriver NPROC = 1 BIN_PATH = INPUT_FILES = @@ -26,7 +26,7 @@ SIMULATION_MODE = NORMAL [hello_world] CLASS = workers SUB_CLASS = - NAME = hello_worker + NAME = HelloWorker NPROC = 1 BIN_PATH = INPUT_FILES = diff --git a/examples-proposed/001-helloworld/helloworld/hello_driver.py b/examples-proposed/001-helloworld/helloworld/hello_driver.py index a4b6bfe3..53aa8839 100644 --- a/examples-proposed/001-helloworld/helloworld/hello_driver.py +++ b/examples-proposed/001-helloworld/helloworld/hello_driver.py @@ -1,7 +1,7 @@ from ipsframework import Component -class hello_driver(Component): +class HelloDriver(Component): """ The IPS framework will always call into helloworld.hello_driver initially, as this module and name (helloworld.hello_driver.hello_driver) were defined in helloworld.conf diff --git a/examples-proposed/001-helloworld/helloworld/hello_worker.py b/examples-proposed/001-helloworld/helloworld/hello_worker.py index 3ba16f5b..24469346 100644 --- a/examples-proposed/001-helloworld/helloworld/hello_worker.py +++ b/examples-proposed/001-helloworld/helloworld/hello_worker.py @@ -1,7 +1,7 @@ from ipsframework import Component -class hello_worker(Component): +class HelloWorker(Component): def __init__(self, services, config): """ Automatically called from the IPS framework diff --git a/examples-proposed/001-helloworld/setup.py b/examples-proposed/001-helloworld/setup.py index 247a04c8..ec4ee7d5 100644 --- a/examples-proposed/001-helloworld/setup.py +++ b/examples-proposed/001-helloworld/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 from setuptools import find_packages, setup setup( diff --git a/examples-proposed/002-same-timestep/driver.py b/examples-proposed/002-same-timestep/driver.py index ae845832..a9660292 100644 --- a/examples-proposed/002-same-timestep/driver.py +++ b/examples-proposed/002-same-timestep/driver.py @@ -3,7 +3,7 @@ from ipsframework import Component -class driver(Component): +class Driver(Component): """Note that only one worker component is called at a time, so the workers are not called in parallel.""" def step(self, timestamp=0.0, **keywords): diff --git a/examples-proposed/002-same-timestep/trace.conf b/examples-proposed/002-same-timestep/trace.conf index 0298c141..daeb4765 100644 --- a/examples-proposed/002-same-timestep/trace.conf +++ b/examples-proposed/002-same-timestep/trace.conf @@ -13,7 +13,7 @@ SIMULATION_MODE = NORMAL [DRIVER] CLASS = DRIVER SUB_CLASS = - NAME = driver + NAME = Driver BIN_PATH = NPROC = 1 INPUT_FILES = @@ -22,9 +22,9 @@ SIMULATION_MODE = NORMAL [WORKER] CLASS = WORKER SUB_CLASS = - NAME = simple_sleep + NAME = SimpleSleep NPROC = 1 - BIN_PATH = + BIN_PATH = INPUT_FILES = OUTPUT_FILES = SCRIPT = $PWD/worker.py diff --git a/examples-proposed/002-same-timestep/worker.py b/examples-proposed/002-same-timestep/worker.py index a8bfbacc..45928eab 100644 --- a/examples-proposed/002-same-timestep/worker.py +++ b/examples-proposed/002-same-timestep/worker.py @@ -6,7 +6,9 @@ from ipsframework import Component -class simple_sleep(Component): +class SimpleSleep(Component): def step(self, timestamp=0.0, **keywords): this_dir = self.services.get_config_param('SIM_ROOT') - self.services.wait_task(self.services.launch_task(1, this_dir, f'{this_dir}{os.path.sep}myscript', 1)) + self.services.wait_task( + self.services.launch_task(1, this_dir, f'{this_dir}{os.path.sep}myscript', 1) + ) diff --git a/examples-proposed/003-different-timestep/driver.py b/examples-proposed/003-different-timestep/driver.py index 0d30fdbd..5c4f5dbe 100644 --- a/examples-proposed/003-different-timestep/driver.py +++ b/examples-proposed/003-different-timestep/driver.py @@ -3,7 +3,7 @@ from ipsframework import Component -class driver(Component): +class Driver(Component): def step(self, timestamp=0.0, **keywords): w = self.services.get_port('WORKER') # call the same worker step twice to check that the trace is correct diff --git a/examples-proposed/003-different-timestep/trace.conf b/examples-proposed/003-different-timestep/trace.conf index b96e5caa..527ed4a4 100644 --- a/examples-proposed/003-different-timestep/trace.conf +++ b/examples-proposed/003-different-timestep/trace.conf @@ -13,7 +13,7 @@ SIMULATION_MODE = NORMAL [DRIVER] CLASS = DRIVER SUB_CLASS = - NAME = driver + NAME = Driver BIN_PATH = NPROC = 1 INPUT_FILES = @@ -22,9 +22,9 @@ SIMULATION_MODE = NORMAL [WORKER] CLASS = WORKER SUB_CLASS = - NAME = simple_sleep + NAME = SimpleSleep NPROC = 1 - BIN_PATH = + BIN_PATH = INPUT_FILES = OUTPUT_FILES = SCRIPT = $PWD/worker.py diff --git a/examples-proposed/003-different-timestep/worker.py b/examples-proposed/003-different-timestep/worker.py index a908c338..205c68b0 100644 --- a/examples-proposed/003-different-timestep/worker.py +++ b/examples-proposed/003-different-timestep/worker.py @@ -6,7 +6,11 @@ from ipsframework import Component -class simple_sleep(Component): +class SimpleSleep(Component): def step(self, timestamp: float, script_name: str): this_dir = self.services.get_config_param('SIM_ROOT') - self.services.wait_task(self.services.launch_task(1, this_dir, f'{this_dir}{os.path.sep}{script_name}', int(timestamp))) + self.services.wait_task( + self.services.launch_task( + 1, this_dir, f'{this_dir}{os.path.sep}{script_name}', int(timestamp) + ) + ) diff --git a/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/bokeh-plots.ipynb b/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/bokeh-plots.ipynb index 4cc6d46a..d11dfab3 100644 --- a/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/bokeh-plots.ipynb +++ b/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/bokeh-plots.ipynb @@ -54,7 +54,13 @@ "\n", " for idx, prop in enumerate(paths):\n", " y = [get_data(d, prop) for d in DATA]\n", - " graph.line(x, y, line_color=COLORS[idx % len(COLORS)], line_dash='solid', legend_label='_'.join(prop))\n", + " graph.line(\n", + " x,\n", + " y,\n", + " line_color=COLORS[idx % len(COLORS)],\n", + " line_dash='solid',\n", + " legend_label='_'.join(prop),\n", + " )\n", " show(graph)" ] } diff --git a/examples-proposed/007-jupyter-child-runs/child_workflow_driver.py b/examples-proposed/007-jupyter-child-runs/child_workflow_driver.py index b1a59c0c..dd3f31b1 100644 --- a/examples-proposed/007-jupyter-child-runs/child_workflow_driver.py +++ b/examples-proposed/007-jupyter-child-runs/child_workflow_driver.py @@ -25,9 +25,9 @@ def step(self, timestamp=0.0, **keywords): def finalize(self, timestamp=0.0, **keywords): """Write the final state to the destination file for the parent component to read, and save the final data""" - OUTPUT_LOCATION = 'analysis.json' - with open(OUTPUT_LOCATION, 'w') as f: + output_location = 'analysis.json' + with open(output_location, 'w') as f: json.dump(self.cache, f) - self.services.add_analysis_data_files([OUTPUT_LOCATION]) + self.services.add_analysis_data_files([output_location]) self.services.info('finalize') diff --git a/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1.py b/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1.py index 6a316150..bf18f3d7 100644 --- a/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1.py +++ b/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1.py @@ -16,7 +16,7 @@ THIS_DIR = Path(__file__).resolve().parent -ips_analysis_data_LIST_FILE = 'ips_analysis_api_data_listing.json' +IPS_ANALYSIS_DATA_LIST_FILE = 'ips_analysis_api_data_listing.json' IPS_CHILD_RUNS_FILE = 'ips_analysis_api_child_runs.txt' @@ -67,7 +67,7 @@ def _normalize_data_filepaths(base_dir: Path, data: dict[str, list[str]]) -> dic def _get_data_from_directory(directory: Path) -> dict[float, list[str]]: """'directory' should be an absolute path, not a relative path.""" - with open(directory / ips_analysis_data_LIST_FILE, 'rb') as f: + with open(directory / IPS_ANALYSIS_DATA_LIST_FILE, 'rb') as f: data: dict[str, list[str]] = json.load(f) return _normalize_data_filepaths(directory, data) diff --git a/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1_notebook.ipynb b/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1_notebook.ipynb index fdac2f67..160a6c25 100644 --- a/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1_notebook.ipynb +++ b/examples-proposed/008-jupyter-multiple-runs/ips_analysis_api_v1_notebook.ipynb @@ -36,7 +36,9 @@ " display(f'Generating tar file from runids: {runids}')\n", "\n", " file = Path(api_v1.generate_tar_from_runids(runids))\n", - " display(f'Generated tar file {file.name} in directory {file.parent}, right click the file in the file browser to download it')\n", + " display(\n", + " f'Generated tar file {file.name} in directory {file.parent}, right click the file in the file browser to download it'\n", + " )\n", "\n", "\n", "widget2.on_click(generate_tarfile)\n", diff --git a/examples-proposed/009-task-pool-sync/dask_worker.py b/examples-proposed/009-task-pool-sync/dask_worker.py index 797b6d0b..767a0329 100644 --- a/examples-proposed/009-task-pool-sync/dask_worker.py +++ b/examples-proposed/009-task-pool-sync/dask_worker.py @@ -5,8 +5,8 @@ from ipsframework import Component -def myFun(*args): - print(f'myFun({args[0]})') +def my_fun(*args): + print(f'my_fun({args[0]})') sleep(float(args[0])) print('function execution completed') return 0 @@ -21,8 +21,8 @@ def step(self, timestamp=0.0): # we only have to wait for half a second once we call submit_tasks() duration = 0.5 self.services.add_task('pool', 'binary', 1, cwd, self.EXECUTABLE, duration) - self.services.add_task('pool', 'function', 1, cwd, myFun, duration) - self.services.add_task('pool', 'method', 1, cwd, copy.copy(self).myMethod, duration) + self.services.add_task('pool', 'function', 1, cwd, my_fun, duration) + self.services.add_task('pool', 'method', 1, cwd, copy.copy(self).my_method, duration) ret_val = self.services.submit_tasks('pool', use_dask=True, dask_nodes=1) print('ret_val =', ret_val, file=stderr) @@ -32,8 +32,8 @@ def step(self, timestamp=0.0): exit_status = self.services.get_finished_tasks('pool') print('exit_status = ', exit_status, file=stderr) - def myMethod(self, *args): - print(f'myMethod({args[0]})') + def my_method(self, *args): + print(f'my_method({args[0]})') sleep(float(args[0])) print('method execution completed') return 0 diff --git a/examples-proposed/020-simple-ensemble/README.md b/examples-proposed/020-simple-ensemble/README.md index 00f2d958..2bdc970a 100644 --- a/examples-proposed/020-simple-ensemble/README.md +++ b/examples-proposed/020-simple-ensemble/README.md @@ -1,4 +1,4 @@ -# Simple ensemble example +# Simple ensemble example This shows how to run an ensemble for three instances. @@ -22,13 +22,13 @@ ignored. These are due to Dask not having a clean shutdown. ## Instructions -### Bash +### Bash To run the code, run: ```bash ips.py --platform platform.conf --simulation ensemble.conf ``` -__NOTE__: disable Global Protect VPN if you are using it, as it can interfere +__NOTE__: disable Global Protect VPN if you are using it, as it can interfere with Dask. @@ -36,15 +36,15 @@ with Dask. To run on Perlmutter, follow these steps -1. Create a conda environment +1. Create a conda environment 2. Install IPS into the conda environment -3. Modify `perlmutter.slurm` to point to your conda environment and to use +3. Modify `perlmutter.slurm` to point to your conda environment and to use your project ID 4. Run the example using the following command from within the directory: ```bash sbatch perlmutter.slurm ``` - + ## Output Running the example will generate log files in the current directory. However, @@ -72,7 +72,7 @@ ENSEMBLES/ │   │   │   └── instance_driver.py │   │   └── work │   │   ├── DRIVER__InstanceDriver_1 - │   │   ├── FWK_COMP_runspaceInitComponent_3 + │   │   ├── FWK_COMP_RunspaceInitComponent_3 │   │   └── WORKER__InstanceComponent_2 │   │   └── stats.csv │   ├── INSTANCE_1 @@ -87,7 +87,7 @@ ENSEMBLES/ │   │   │   └── instance_driver.py │   │   └── work │   │   ├── DRIVER__InstanceDriver_1 - │   │   ├── FWK_COMP_runspaceInitComponent_3 + │   │   ├── FWK_COMP_RunspaceInitComponent_3 │   │   └── WORKER__InstanceComponent_2 │   │   └── stats.csv │   ├── INSTANCE_2 @@ -105,18 +105,18 @@ ENSEMBLES/ Observe that the instances have their own directories under `work/DRIVER__EnsembleDriver_1/` -and that they follow the naming pattern of `INSTANCE_`; this -pattern was given with the `name` parameter in the `run_ensemble()` call +and that they follow the naming pattern of `INSTANCE_`; this +pattern was given with the `name` parameter in the `run_ensemble()` call found in `driver.py`, which is the top-level driving component. Inside each -instance directory is the output from the IPS run for that instance. Further note that inside the `work` directory for each instance under -`WORKER__InstanceComponent_2` is a `stats.csv` file that contains the output -from that instance's component. This file contains the instance name (e.g., +instance directory is the output from the IPS run for that instance. Further note that inside the `work` directory for each instance under +`WORKER__InstanceComponent_2` is a `stats.csv` file that contains the output +from that instance's component. This file contains the instance name (e.g., `INSTANCE_0`), the IPS executable script, the hostname -where the component ran, the process id, core ID, and the start and end times +where the component ran, the process id, core ID, and the start and end times that component ran. Note that all three components ran at about the same time on different cores. -You can conveniently see all the `stats.csv` files for all instances by running +You can conveniently see all the `stats.csv` files for all instances by running the following command: ```bash @@ -127,4 +127,3 @@ find . -name stats.csv | xargs cat The next example is 021-ensembles-from-CSV, which shows how to run an ensemble where the instance parameters are read from a CSV file. - diff --git a/examples-proposed/020-simple-ensemble/driver.py b/examples-proposed/020-simple-ensemble/driver.py index a634e408..2a98d6b9 100644 --- a/examples-proposed/020-simple-ensemble/driver.py +++ b/examples-proposed/020-simple-ensemble/driver.py @@ -1,14 +1,14 @@ -#!/usr/bin/env python3 """ - Simple ensemble driver that just dispatches an IPS ensemble. +Simple ensemble driver that just dispatches an IPS ensemble. """ + from pathlib import Path from ipsframework import Component class EnsembleDriver(Component): - """ Kicks off a simple ensemble """ + """Kicks off a simple ensemble""" def step(self, timestamp=0.0): # Specifies different sets of variable values for concurrent ensemble @@ -34,11 +34,12 @@ def step(self, timestamp=0.0): # be defined in a special template IPS configuration file, in this case, # `template.conf`. variables = { - 'instance_component': { - 'A': [3, 2, 4], - 'B': [2.34, 5.82, 0.1], - 'C': ['bar', 'baz', 'quux']}} - + 'instance_component': { + 'A': [3, 2, 4], + 'B': [2.34, 5.82, 0.1], + 'C': ['bar', 'baz', 'quux'], + } + } # This is the IPS configuration file for the instances that looks like # a regular configuration file except there are slots for the 'A', 'B', # and 'C' for variable substitution. 'TEMPLATE' is specified in the @@ -58,13 +59,16 @@ def step(self, timestamp=0.0): # We also demonstrate that stdout and stderr output per instance can # be captured in files by specifying logfile and errfile, respectively. - mapping = self.services.run_ensemble(template, variables, - run_dir=Path('.').absolute(), - name='INSTANCE_', - num_nodes=1, - cores_per_instance=1, - logfile='logfile.txt', - errfile='errfile.txt') + mapping = self.services.run_ensemble( + template, + variables, + run_dir=Path('.').absolute(), + name='INSTANCE_', + num_nodes=1, + cores_per_instance=1, + logfile='logfile.txt', + errfile='errfile.txt', + ) # Print each mapping of instance name to what variable values were used. for instance in mapping: self.services.info(f'{instance!s}') diff --git a/examples-proposed/020-simple-ensemble/instance_component.py b/examples-proposed/020-simple-ensemble/instance_component.py index 2d017db0..28d339c6 100644 --- a/examples-proposed/020-simple-ensemble/instance_component.py +++ b/examples-proposed/020-simple-ensemble/instance_component.py @@ -1,36 +1,32 @@ -#!/usr/bin/env python3 """ - Component to be stepped in instance +Component to be stepped in instance """ -import os -import sys + import csv +import sys from time import time from ipsframework import Component -from ipsframework.resourceHelper import get_platform_info +from ipsframework.resource_helper import get_platform_info class InstanceComponent(Component): - def step(self, timestamp: float = 0.0, **keywords): start = time() - - # ENSEMBLE_INSTANCE is a special IPS variable that contains the # string uniquely identifying this instance. Each instance will have # the `run_ensemble()` `name` argument prepended to a unique number # for each instance. E.g., ENSEMBLE_INSTANCE might be "MY_INSTANCE_23". instance_id = self.services.get_config_param('ENSEMBLE_INSTANCE') - self.services.info(f'{instance_id}: Start of step of instance ' - f'component.') + self.services.info(f'{instance_id}: Start of step of instance component.') print(f'start of instance component for {instance_id}') # Echo the parameters we're expecting, A, B, and C - self.services.info(f'{instance_id}: instance component parameters: ' - f'A={self.A}, B={self.B}, C={self.C}') + self.services.info( + f'{instance_id}: instance component parameters: A={self.A}, B={self.B}, C={self.C}' + ) print(f'Values for {instance_id}: A={self.A}, B={self.B}, C={self.C}') @@ -39,12 +35,19 @@ def step(self, timestamp: float = 0.0, **keywords): with open('stats.csv', 'w') as f: writer = csv.writer(f) - writer.writerow(['instance', 'executable', 'hostname', 'pid', - 'core', 'start', 'end']) - writer.writerow([instance_id, sys.argv[0], run_env['hostname'], - run_env['pid'], run_env['core_id'], start, time()]) + writer.writerow(['instance', 'executable', 'hostname', 'pid', 'core', 'start', 'end']) + writer.writerow( + [ + instance_id, + sys.argv[0], + run_env['hostname'], + run_env['pid'], + run_env['core_id'], + start, + time(), + ] + ) print(f'Wrote stats.csv for {instance_id}') - self.services.info(f'{instance_id}: End of step of instance ' - f'component.') + self.services.info(f'{instance_id}: End of step of instance component.') diff --git a/examples-proposed/020-simple-ensemble/instance_driver.py b/examples-proposed/020-simple-ensemble/instance_driver.py index cbcc4f16..b2fd7708 100644 --- a/examples-proposed/020-simple-ensemble/instance_driver.py +++ b/examples-proposed/020-simple-ensemble/instance_driver.py @@ -1,17 +1,16 @@ -#!/usr/bin/env python3 """ - Driver component for instances +Driver component for instances """ + from ipsframework import Component class InstanceDriver(Component): """ - Instance driver component that steps the main component + Instance driver component that steps the main component """ def step(self, timestamp: float = 0.0, **keywords): instance_component = self.services.get_port('WORKER') self.services.call(instance_component, 'step', 0.0) - diff --git a/examples-proposed/021-ensembles-from-CSV/driver.py b/examples-proposed/021-ensembles-from-CSV/driver.py index 5ef85fdc..dda18dcb 100644 --- a/examples-proposed/021-ensembles-from-CSV/driver.py +++ b/examples-proposed/021-ensembles-from-CSV/driver.py @@ -1,14 +1,15 @@ -#!/usr/bin/env python3 """ - Simple ensemble driver that just dispatches an IPS ensemble. +Simple ensemble driver that just dispatches an IPS ensemble. """ + from pathlib import Path from ipsframework import Component from ipsframework.ipsutil import params_from_csv + class EnsembleDriver(Component): - """ Kicks off an ensemble using variables from a CSV file. """ + """Kicks off an ensemble using variables from a CSV file.""" def step(self, timestamp=0.0): # Read in the variable combinations from a CSV file. The CSV file @@ -44,12 +45,15 @@ def step(self, timestamp=0.0): # for each instance to `my_logfile.txt`. I.e., by *not* specifying # a filename for `errfile`, the stdout and stderr are combined into a # single file for each instance. - mapping = self.services.run_ensemble(template, variables, - run_dir=Path('.').absolute(), - name='INSTANCE_', - num_nodes=1, - cores_per_instance=1, - logfile='my_logfile.txt') + mapping = self.services.run_ensemble( + template, + variables, + run_dir=Path('.').absolute(), + name='INSTANCE_', + num_nodes=1, + cores_per_instance=1, + logfile='my_logfile.txt', + ) # Print each mapping of instance name to what variable values were used. for instance in mapping: - self.services.info(f'{instance!s}') \ No newline at end of file + self.services.info(f'{instance!s}') diff --git a/examples-proposed/021-ensembles-from-CSV/instance_component.py b/examples-proposed/021-ensembles-from-CSV/instance_component.py index 7c440d7d..34f51610 100644 --- a/examples-proposed/021-ensembles-from-CSV/instance_component.py +++ b/examples-proposed/021-ensembles-from-CSV/instance_component.py @@ -1,18 +1,16 @@ -#!/usr/bin/env python3 """ - Component to be stepped in instance +Component to be stepped in instance """ -import os -import sys + import csv +import sys from time import time from ipsframework import Component -from ipsframework.resourceHelper import get_platform_info +from ipsframework.resource_helper import get_platform_info class InstanceComponent(Component): - def step(self, timestamp: float = 0.0, **keywords): start = time() @@ -21,12 +19,12 @@ def step(self, timestamp: float = 0.0, **keywords): # the `run_ensemble()` `name` argument prepended to a unique number # for each instance. E.g., ENSEMBLE_INSTANCE might be "MY_INSTANCE_23". instance_id = self.services.get_config_param('ENSEMBLE_INSTANCE') - self.services.info(f'{instance_id}: Start of step of instance ' - f'component.') + self.services.info(f'{instance_id}: Start of step of instance component.') # Echo the parameters we're expecting, A, B, and C - self.services.info(f'{instance_id}: instance component parameters: ' - f'A={self.A}, B={self.B}, C={self.C}') + self.services.info( + f'{instance_id}: instance component parameters: A={self.A}, B={self.B}, C={self.C}' + ) print(f'{instance_id}: A={self.A}, B={self.B}, C={self.C}') @@ -35,11 +33,17 @@ def step(self, timestamp: float = 0.0, **keywords): with open('stats.csv', 'w') as f: writer = csv.writer(f) - writer.writerow(['instance', 'executable', 'hostname', 'pid', - 'core', 'start', 'end']) - writer.writerow([instance_id, sys.argv[0], run_env['hostname'], - run_env['pid'], run_env['core_id'], start, time()]) - - - self.services.info(f'{instance_id}: End of step of instance ' - f'component.') + writer.writerow(['instance', 'executable', 'hostname', 'pid', 'core', 'start', 'end']) + writer.writerow( + [ + instance_id, + sys.argv[0], + run_env['hostname'], + run_env['pid'], + run_env['core_id'], + start, + time(), + ] + ) + + self.services.info(f'{instance_id}: End of step of instance component.') diff --git a/examples-proposed/021-ensembles-from-CSV/instance_driver.py b/examples-proposed/021-ensembles-from-CSV/instance_driver.py index cbcc4f16..b2fd7708 100644 --- a/examples-proposed/021-ensembles-from-CSV/instance_driver.py +++ b/examples-proposed/021-ensembles-from-CSV/instance_driver.py @@ -1,17 +1,16 @@ -#!/usr/bin/env python3 """ - Driver component for instances +Driver component for instances """ + from ipsframework import Component class InstanceDriver(Component): """ - Instance driver component that steps the main component + Instance driver component that steps the main component """ def step(self, timestamp: float = 0.0, **keywords): instance_component = self.services.get_port('WORKER') self.services.call(instance_component, 'step', 0.0) - diff --git a/examples-proposed/022-tasks-and-ensembles/driver.py b/examples-proposed/022-tasks-and-ensembles/driver.py index 4e4e858c..435dc1e3 100644 --- a/examples-proposed/022-tasks-and-ensembles/driver.py +++ b/examples-proposed/022-tasks-and-ensembles/driver.py @@ -1,14 +1,15 @@ -#!/usr/bin/env python3 """ - Simple ensemble driver that just dispatches an IPS ensemble. +Simple ensemble driver that just dispatches an IPS ensemble. """ + from pathlib import Path from ipsframework import Component from ipsframework.ipsutil import params_from_csv + class EnsembleDriver(Component): - """ Kicks off an ensemble using variables from a CSV file. """ + """Kicks off an ensemble using variables from a CSV file.""" def step(self, timestamp=0.0): # Read in the variable combinations from a CSV file. The CSV file @@ -43,14 +44,16 @@ def step(self, timestamp=0.0): # `INSTANCE_1` subdirectory. # NOTE: we are requesting 2 cores per instance here to match the MPI # executable used in the instance component. - mapping = self.services.run_ensemble(template, variables, - run_dir=Path('.').absolute(), - name='INSTANCE_', - num_nodes=1, - cores_per_instance=5, - oversubscribe=False, - logfile='logfile.txt') + mapping = self.services.run_ensemble( + template, + variables, + run_dir=Path('.').absolute(), + name='INSTANCE_', + num_nodes=1, + cores_per_instance=5, + oversubscribe=False, + logfile='logfile.txt', + ) # Print each mapping of instance name to what variable values were used. for instance in mapping: self.services.info(f'{instance!s}') - diff --git a/examples-proposed/022-tasks-and-ensembles/instance_component.py b/examples-proposed/022-tasks-and-ensembles/instance_component.py index 1eb8c32d..b0cfb888 100644 --- a/examples-proposed/022-tasks-and-ensembles/instance_component.py +++ b/examples-proposed/022-tasks-and-ensembles/instance_component.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 """ - Component to be stepped in instance +Component to be stepped in instance """ + import os from pathlib import Path @@ -9,24 +9,23 @@ class InstanceComponent(Component): - def step(self, timestamp: float = 0.0, **keywords): if 'HWLOC_XMLFILE' in os.environ: - self.services.warning(f'HWLOC_XMLfile still set!') + self.services.warning('HWLOC_XMLfile still set!') else: self.services.info('HWLOC_XMLFILE is not set') - + # ENSEMBLE_INSTANCE is a special IPS variable that contains the # string uniquely identifying this instance. Each instance will have # the `run_ensemble()` `name` argument prepended to a unique number # for each instance. E.g., ENSEMBLE_INSTANCE might be "MY_INSTANCE_23". instance_id = self.services.get_config_param('ENSEMBLE_INSTANCE') - self.services.info(f'{instance_id}: Start of step of instance ' - f'component.') + self.services.info(f'{instance_id}: Start of step of instance component.') # Echo the parameters we're expecting, A, B, and C - self.services.info(f'{instance_id}: instance component parameters: ' - f'A={self.A}, B={self.B}, C={self.C}') + self.services.info( + f'{instance_id}: instance component parameters: A={self.A}, B={self.B}, C={self.C}' + ) # We set the MPI executable path in the environment variable # MPI_STATS_EXEC in the `perlmutter.slurm` script that launches this @@ -35,23 +34,25 @@ def step(self, timestamp: float = 0.0, **keywords): # script. mpi_executable = os.environ['MPI_STATS_EXEC'] working_dir = str(Path('.').absolute()) - self.services.info(f'{instance_id}: Launching MPI executable ' - f'{mpi_executable} in {working_dir}') - args = ['-i', instance_id, - '-s', str(self.B), # arbitrarily using B to specify sleep time - '-o', 'stats.csv'] + self.services.info( + f'{instance_id}: Launching MPI executable {mpi_executable} in {working_dir}' + ) + args = [ + '-i', + instance_id, + '-s', + str(self.B), # arbitrarily using B to specify sleep time + '-o', + 'stats.csv', + ] cmd = str(mpi_executable) + ' ' + ' '.join(args) try: - run_id = self.services.launch_task(nproc=5, - working_dir=working_dir, - binary=cmd) - except Exception as e: - self.services.critical(f'{instance_id}: Unable to launch ' - f'{mpi_executable}') + run_id = self.services.launch_task(nproc=5, working_dir=working_dir, binary=cmd) + except Exception: + self.services.critical(f'{instance_id}: Unable to launch {mpi_executable}') self.services.wait_task(run_id) # block until done self.services.info(f'{instance_id}: Completed MPI executable.') - self.services.info(f'{instance_id}: End of step of instance ' - f'component.') + self.services.info(f'{instance_id}: End of step of instance component.') diff --git a/examples-proposed/022-tasks-and-ensembles/instance_driver.py b/examples-proposed/022-tasks-and-ensembles/instance_driver.py index cbcc4f16..b2fd7708 100644 --- a/examples-proposed/022-tasks-and-ensembles/instance_driver.py +++ b/examples-proposed/022-tasks-and-ensembles/instance_driver.py @@ -1,17 +1,16 @@ -#!/usr/bin/env python3 """ - Driver component for instances +Driver component for instances """ + from ipsframework import Component class InstanceDriver(Component): """ - Instance driver component that steps the main component + Instance driver component that steps the main component """ def step(self, timestamp: float = 0.0, **keywords): instance_component = self.services.get_port('WORKER') self.services.call(instance_component, 'step', 0.0) - diff --git a/examples-proposed/022-tasks-and-ensembles/mpi_stats.py b/examples-proposed/022-tasks-and-ensembles/mpi_stats.py index fa578697..2ea2021b 100755 --- a/examples-proposed/022-tasks-and-ensembles/mpi_stats.py +++ b/examples-proposed/022-tasks-and-ensembles/mpi_stats.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 -""" Echoes the MPI rank and size of the communicator, the hostname, the PID, - affinity, and the number of cores. Also sleeps for a specified amount of time. +"""Echoes the MPI rank and size of the communicator, the hostname, the PID, +affinity, and the number of cores. Also sleeps for a specified amount of time. """ + import argparse import csv +import os +import socket import sys - -from mpi4py import MPI from time import sleep, time -import os -import socket +from mpi4py import MPI try: import psutil @@ -21,14 +21,9 @@ start = time() parser = argparse.ArgumentParser(description='MPI stats') - parser.add_argument('-i', '--id', type=str, - default=str(os.getpid()), - help='Task ID') - parser.add_argument('-s', '--sleep', - default=5.0, type=float, - help='Sleep time in seconds') - parser.add_argument('-o', '--output', type=str, - default=None, help='Output CSV file') + parser.add_argument('-i', '--id', type=str, default=str(os.getpid()), help='Task ID') + parser.add_argument('-s', '--sleep', default=5.0, type=float, help='Sleep time in seconds') + parser.add_argument('-o', '--output', type=str, default=None, help='Output CSV file') args = parser.parse_args() @@ -51,22 +46,44 @@ affinity = None n_cores = os.cpu_count() # fallback - print(f"Rank {rank} of {size} in task {args.id} on {hostname} " - f"(pid {pid}) affinity {affinity!s} n_cores {n_cores} ") + print( + f'Rank {rank} of {size} in task {args.id} on {hostname} (pid {pid}) affinity {affinity!s} n_cores {n_cores} ' + ) if args.sleep > 0: - print(f"Task {args.id} sleeping {args.sleep} seconds...") + print(f'Task {args.id} sleeping {args.sleep} seconds...') sleep(args.sleep) if args.output is not None: with open(f'rank_{rank}_{args.output}', 'w', newline='') as csvfile: - fieldnames = ['id', 'hostname', 'rank', 'size', 'pid', 'n_cores', 'affinity', 's', 'start', 'end'] + fieldnames = [ + 'id', + 'hostname', + 'rank', + 'size', + 'pid', + 'n_cores', + 'affinity', + 's', + 'start', + 'end', + ] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() - writer.writerow({'id': args.id, 'hostname': hostname, - 'rank': rank, 'size': size, 'pid': pid, - 'n_cores': n_cores, 'affinity': str(affinity), 's': args.sleep, - 'start': start, 'end': time()}) + writer.writerow( + { + 'id': args.id, + 'hostname': hostname, + 'rank': rank, + 'size': size, + 'pid': pid, + 'n_cores': n_cores, + 'affinity': str(affinity), + 's': args.sleep, + 'start': start, + 'end': time(), + } + ) sys.exit(0) diff --git a/examples-proposed/023-simple-ensemble-with-portal/driver.py b/examples-proposed/023-simple-ensemble-with-portal/driver.py index 4c79cb68..632fe1ea 100644 --- a/examples-proposed/023-simple-ensemble-with-portal/driver.py +++ b/examples-proposed/023-simple-ensemble-with-portal/driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Simple ensemble driver that just dispatches an IPS ensemble. """ @@ -12,10 +11,10 @@ class EnsembleDriver(Component): """Kicks off a simple ensemble""" def init(self, timestamp=0.0): - NOTEBOOK_TEMPLATE = 'notebook.ipynb' - self.services.stage_input_files([NOTEBOOK_TEMPLATE]) + notebook_template = 'notebook.ipynb' + self.services.stage_input_files([notebook_template]) try: - self.services.initialize_jupyter_notebook(NOTEBOOK_TEMPLATE) + self.services.initialize_jupyter_notebook(notebook_template) except Exception: print('did not add notebook to portal') @@ -42,7 +41,13 @@ def step(self, timestamp=0.0): # should be identical. Note that placeholders for these variables must # be defined in a special template IPS configuration file, in this case, # `template.conf`. - variables = {'instance_component': {'base_x': [3, 2, 4], 'base_y': [2.34, 5.82, 0.1], 'word': ['bar', 'baz', 'quux']}} + variables = { + 'instance_component': { + 'base_x': [3, 2, 4], + 'base_y': [2.34, 5.82, 0.1], + 'word': ['bar', 'baz', 'quux'], + } + } # This is the IPS configuration file for the instances that looks like # a regular configuration file except there are slots for the 'base_x', 'base_y', @@ -62,7 +67,14 @@ def step(self, timestamp=0.0): # `my_simple_ensemble1` subdirectory. # # The "name" parameter must be unique for each ensemble within a run, and will be used as an identifier on the Portal. - mapping = self.services.run_ensemble(template, variables, run_dir=Path('.').absolute(), name='my_simple_ensemble', num_nodes=1, cores_per_instance=1) + mapping = self.services.run_ensemble( + template, + variables, + run_dir=Path('.').absolute(), + name='my_simple_ensemble', + num_nodes=1, + cores_per_instance=1, + ) # Print each mapping of instance name to what variable values were used. for instance in mapping: self.services.info(f'{instance!s}') diff --git a/examples-proposed/023-simple-ensemble-with-portal/input_dir/notebook.ipynb b/examples-proposed/023-simple-ensemble-with-portal/input_dir/notebook.ipynb index 8c2cba07..0cb356f0 100644 --- a/examples-proposed/023-simple-ensemble-with-portal/input_dir/notebook.ipynb +++ b/examples-proposed/023-simple-ensemble-with-portal/input_dir/notebook.ipynb @@ -1,57 +1,57 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "5d75faa3", - "metadata": {}, - "outputs": [], - "source": [ - "# Notebook template, the IPS Framework will add a cell before this one\n", - "# defining the ips_analysis_api variable.\n", - "\n", - "import json\n", - "\n", - "from bokeh.io import output_notebook\n", - "from bokeh.plotting import figure, show\n", - "\n", - "output_notebook()\n", - "\n", - "plot = figure(width=1000, height=1000)\n", - "\n", - "COLORS = ['red', 'blue', 'green']\n", - "color_idx = 0\n", - "\n", - "# This notebook assumes that all ensembles from this run which save JSON files have a similar data format\n", - "value: dict[int, dict[float, list[str]]] = ips_analysis_api.get_child_data_by_ensemble_names()\n", - "for _runid, runid_map in value.items():\n", - " for _timestep, data_files in runid_map.items():\n", - " for path in filter(lambda fname: fname.endswith('.json'), data_files):\n", - " with open(path, 'r') as f:\n", - " data = json.loads(f.read())\n", - " plot.scatter(\n", - " data['x_data'],\n", - " data['y_data'],\n", - " marker='plus',\n", - " size=5,\n", - " color=COLORS[color_idx],\n", - " alpha=0.5,\n", - " legend_label=f'{data[\"word\"]} ({data[\"base_x\"]}, {data[\"base_y\"]})',\n", - " )\n", - " color_idx = (color_idx + 1) % len(COLORS)\n", - "\n", - "plot.legend.location = 'top_left'\n", - "plot.legend.click_policy = 'hide'\n", - "\n", - "show(plot)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "5d75faa3", + "metadata": {}, + "outputs": [], + "source": [ + "# Notebook template, the IPS Framework will add a cell before this one\n", + "# defining the ips_analysis_api variable.\n", + "\n", + "import json\n", + "\n", + "from bokeh.io import output_notebook\n", + "from bokeh.plotting import figure, show\n", + "\n", + "output_notebook()\n", + "\n", + "plot = figure(width=1000, height=1000)\n", + "\n", + "COLORS = ['red', 'blue', 'green']\n", + "color_idx = 0\n", + "\n", + "# This notebook assumes that all ensembles from this run which save JSON files have a similar data format\n", + "value: dict[int, dict[float, list[str]]] = ips_analysis_api.get_child_data_by_ensemble_names()\n", + "for _runid, runid_map in value.items(): # noqa: PERF102 # show full example\n", + " for _timestep, data_files in runid_map.items(): # noqa: PERF102\n", + " for path in filter(lambda fname: fname.endswith('.json'), data_files):\n", + " with open(path, 'r') as f:\n", + " data = json.loads(f.read())\n", + " plot.scatter(\n", + " data['x_data'],\n", + " data['y_data'],\n", + " marker='plus',\n", + " size=5,\n", + " color=COLORS[color_idx],\n", + " alpha=0.5,\n", + " legend_label=f'{data[\"word\"]} ({data[\"base_x\"]}, {data[\"base_y\"]})',\n", + " )\n", + " color_idx = (color_idx + 1) % len(COLORS)\n", + "\n", + "plot.legend.location = 'top_left'\n", + "plot.legend.click_policy = 'hide'\n", + "\n", + "show(plot)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/examples-proposed/023-simple-ensemble-with-portal/instance_component.py b/examples-proposed/023-simple-ensemble-with-portal/instance_component.py index 14197c41..976571c7 100644 --- a/examples-proposed/023-simple-ensemble-with-portal/instance_component.py +++ b/examples-proposed/023-simple-ensemble-with-portal/instance_component.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Component to be stepped in instance """ @@ -11,7 +10,7 @@ from typing import Any from ipsframework import Component -from ipsframework.resourceHelper import get_platform_info +from ipsframework.resource_helper import get_platform_info def generate_fake_data(timestamp: float, base_x: float, base_y: float, word: str) -> dict[str, Any]: @@ -49,7 +48,9 @@ def step(self, timestamp: float = 0.0, **keywords): self.services.info(f'{instance_id}: Start of step of instance component.') # Echo the parameters we're expecting, A, B, and C - self.services.info(f'{instance_id}: instance component parameters: base_x={self.base_x}, base_y={self.base_y}, word={self.word}') + self.services.info( + f'{instance_id}: instance component parameters: base_x={self.base_x}, base_y={self.base_y}, word={self.word}' + ) # generate some fake data and save it data_fname = f'generated_{timestamp}.json' @@ -64,7 +65,17 @@ def step(self, timestamp: float = 0.0, **keywords): with open(stats_fname, 'w') as f: writer = csv.writer(f) writer.writerow(['instance', 'executable', 'hostname', 'pid', 'core', 'start', 'end']) - writer.writerow([instance_id, sys.argv[0], run_env['hostname'], run_env['pid'], run_env['core_id'], start, time()]) + writer.writerow( + [ + instance_id, + sys.argv[0], + run_env['hostname'], + run_env['pid'], + run_env['core_id'], + start, + time(), + ] + ) try: self.services.add_analysis_data_files([data_fname, stats_fname], timestamp) diff --git a/examples-proposed/023-simple-ensemble-with-portal/instance_driver.py b/examples-proposed/023-simple-ensemble-with-portal/instance_driver.py index 33f1954b..b2fd7708 100644 --- a/examples-proposed/023-simple-ensemble-with-portal/instance_driver.py +++ b/examples-proposed/023-simple-ensemble-with-portal/instance_driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Driver component for instances """ diff --git a/examples-proposed/024-aggregated-compute-ensemble/driver.py b/examples-proposed/024-aggregated-compute-ensemble/driver.py index 2b0c4d76..495a69a0 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/driver.py +++ b/examples-proposed/024-aggregated-compute-ensemble/driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Simple ensemble driver that just dispatches an IPS ensemble for an example compute application. @@ -10,7 +9,7 @@ from ipsframework.ipsutil import params_from_csv # The notebook that will be copied for each ensemble instance -SOURCE_NOTEBOOK_NAME='global_notebook.ipynb' +SOURCE_NOTEBOOK_NAME = 'global_notebook.ipynb' class EnsembleDriver(Component): @@ -20,11 +19,10 @@ def init(self, timestamp=0.0): self.services.stage_input_files([SOURCE_NOTEBOOK_NAME]) self.services.initialize_jupyter_notebook( - dest_notebook_name='jupyterhub_global_notebook.ipynb', - source_notebook_path=SOURCE_NOTEBOOK_NAME, + dest_notebook_name='jupyterhub_global_notebook.ipynb', + source_notebook_path=SOURCE_NOTEBOOK_NAME, ) - def step(self, timestamp=0.0): # This CSV file contains the parameters used for the # different instances. @@ -38,8 +36,7 @@ def step(self, timestamp=0.0): self.services.info(f'Using template config file {template}') if not template.exists(): - raise RuntimeError( - f'{template} config template file does not exist') + raise RuntimeError(f'{template} config template file does not exist') # Now spin up and run the instances. This function will return a list # with each list element corresponding to an instance. You can use @@ -48,10 +45,14 @@ def step(self, timestamp=0.0): # # The "name" parameter must be unique for each ensemble within a run, # and will be used as an identifier on the Portal. - mapping = self.services.run_ensemble(template, variables, - run_dir=Path('.').absolute(), - name='INSTANCE_', - num_nodes=1, cores_per_instance=1) + mapping = self.services.run_ensemble( + template, + variables, + run_dir=Path('.').absolute(), + name='INSTANCE_', + num_nodes=1, + cores_per_instance=1, + ) # Print each mapping of instance name to what variable values were used. for instance in mapping: diff --git a/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf b/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf index c41c8bd2..79aa7cac 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf +++ b/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf @@ -8,8 +8,7 @@ SIMULATION_MODE = NORMAL INPUT_DIR = $PWD/input_dir/ USE_PORTAL = True -PORTAL_URL = http://localhost:5000 -# PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org +PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org # do not commit actual PORTAL_API_KEY value to version control, best to set as an environment variable #PORTAL_API_KEY=changeme @@ -32,4 +31,3 @@ PORTAL_URL = http://localhost:5000 TEMPLATE = $PWD/template.conf # Specifies the parameter values for each instance PARAMETER_FILE = $PWD/values.csv - diff --git a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py index 0c9f5bf0..971c7b4f 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py +++ b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py @@ -1,72 +1,69 @@ -#!/usr/bin/env python3 """ - Used to generate synthetic data as an example. +Used to generate synthetic data as an example. - Writes two output files: +Writes two output files: - * `_solution.json`: contains the synthetic data - * `_stats.csv`: contains provenance information about the run +* `_solution.json`: contains the synthetic data +* `_stats.csv`: contains provenance information about the run - The JSON file is, in turn, read by the a per-instance jupyter notebook - available on the Portal to generate a plot of the data. +The JSON file is, in turn, read by the a per-instance jupyter notebook +available on the Portal to generate a plot of the data. """ + import argparse -from typing import Any -import json import csv +import json from time import time from traceback import print_exc +from typing import Any + import numpy as np -from ipsframework.resourceHelper import get_platform_info +from ipsframework.resource_helper import get_platform_info -def main(instance: str, - alpha: float, L:float, T_final:float, Nx:int, Nt:int) -> dict[str, Any]: - """ Generate synthetic data to emulate an actual simulation or complex - calculation. +def main(instance: str, alpha: float, l: float, t_final: float, nx: int, nt: int) -> dict[str, Any]: + """Generate synthetic data to emulate an actual simulation or complex + calculation. - As a side-effect it will save a plot to the current working directory with - the name `solution.png`. + As a side-effect it will save a plot to the current working directory with + the name `solution.png`. - :param instance: instance name - :param alpha: thermal diffusivity - :param L: domain length - :param T_final: final time - :param Nx: number of spatial grid points - :param Nt: number of time steps - :returns: x, y, where x is the steps and u the corresponding values + :param instance: instance name + :param alpha: thermal diffusivity + :param l: domain length + :param t_final: final time + :param nx: number of spatial grid points + :param nt: number of time steps + :returns: x, y, where x is the steps and u the corresponding values """ start = time() # Discretization - dx = L / (Nx - 1) - dt = T_final / Nt - r = alpha * dt / (dx ** 2) + dx = l / (nx - 1) + dt = t_final / nt + r = alpha * dt / (dx**2) # # Check stability condition for explicit method if r > 0.5: - print("Warning: Stability condition r <= 0.5 is not met. " - "Results may be inaccurate.") + print('Warning: Stability condition r <= 0.5 is not met. Results may be inaccurate.') # Initial condition (e.g., a sine wave) - x = np.linspace(0, L, Nx) + x = np.linspace(0, l, nx) u = np.sin(np.pi * x) - # Boundary conditions (Dirichlet, e.g., u(0,t) = 0, u(L,t) = 0) These are + # Boundary conditions (Dirichlet, e.g., u(0,t) = 0, u(l,t) = 0) These are # already handled by the initial setup of u=0 at boundaries if the # initial condition is 0 there. If non-zero, they would be set within the # time loop. # Time evolution - for n in range(Nt): + for _n in range(nt): u_new = np.copy(u) # Create a copy for updating - for i in range(1, Nx - 1): + for i in range(1, nx - 1): u_new[i] = u[i] + r * (u[i + 1] - 2 * u[i] + u[i - 1]) u = u_new - - # Save some per-component stats stats_fname = f'{instance}_stats.csv' run_env = get_platform_info() @@ -76,38 +73,57 @@ def main(instance: str, # specific to this instance. writer = csv.writer(f) writer.writerow( - ['instance', 'hostname', 'pid', 'core', - 'affinity', - 'alpha', 'L', 'T_final', 'Nx', 'Nt', - 'start', 'end']) + [ + 'instance', + 'hostname', + 'pid', + 'core', + 'affinity', + 'alpha', + 'l', + 't_final', + 'nx', + 'nt', + 'start', + 'end', + ] + ) - writer.writerow([instance, run_env['hostname'], - run_env['pid'], run_env['core_id'], - run_env['affinity'], - alpha, L, T_final, Nx, Nt, - start, time()]) + writer.writerow( + [ + instance, + run_env['hostname'], + run_env['pid'], + run_env['core_id'], + run_env['affinity'], + alpha, + l, + t_final, + nx, + nt, + start, + time(), + ] + ) return {'x': x.tolist(), 'u': u.tolist()} - if __name__ == '__main__': try: - parser = argparse.ArgumentParser(description='Generate synthetic data to ' - 'emulate an actual simulation ' - 'or complex') - parser.add_argument('--instance', type=str, - help='instance name') - parser.add_argument('--alpha', type=float, default=1.0,) - parser.add_argument('--L', type=float, default=1.0,) - parser.add_argument('--T_final', type=float, default=1.0,) - parser.add_argument('--Nx', type=int, default=100,) - parser.add_argument('--Nt', type=int, default=100,) + parser = argparse.ArgumentParser( + description='Generate synthetic data to emulate an actual simulation or complex' + ) + parser.add_argument('--instance', type=str, help='instance name') + parser.add_argument('--alpha', type=float, default=1.0) + parser.add_argument('--l', type=float, default=1.0) + parser.add_argument('--t_final', type=float, default=1.0) + parser.add_argument('--nx', type=int, default=100) + parser.add_argument('--nt', type=int, default=100) args = parser.parse_args() - data = main(args.instance, - args.alpha, args.L, args.T_final, args.Nx, args.Nt) + data = main(args.instance, args.alpha, args.l, args.t_final, args.nx, args.nt) file_name = f'{args.instance}_solution.json' print(f'Writing to {file_name}') diff --git a/examples-proposed/024-aggregated-compute-ensemble/input_dir/global_notebook.ipynb b/examples-proposed/024-aggregated-compute-ensemble/input_dir/global_notebook.ipynb index 52c791ee..3231c775 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/input_dir/global_notebook.ipynb +++ b/examples-proposed/024-aggregated-compute-ensemble/input_dir/global_notebook.ipynb @@ -14,66 +14,71 @@ ] }, { + "cell_type": "code", + "execution_count": 2, + "id": "857879d9069625cf", "metadata": { "ExecuteTime": { "end_time": "2026-01-21T15:24:56.068204Z", "start_time": "2026-01-21T15:24:56.059637Z" } }, - "cell_type": "code", + "outputs": [], "source": [ - "import json\n", "import csv\n", "from pathlib import Path\n", - "import pandas as pd\n", "\n", - "import matplotlib.pyplot as plt" - ], - "id": "857879d9069625cf", - "outputs": [], - "execution_count": 2 + "import pandas as pd" + ] }, { + "cell_type": "code", + "execution_count": 1, + "id": "3d8fa1127a2ab4ef", "metadata": { "ExecuteTime": { "end_time": "2026-01-21T15:24:48.049301Z", "start_time": "2026-01-21T15:24:48.038184Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "# Gather ensemble instance data\n", "ensemble_data = ips_analysis_api.get_child_data()" - ], - "id": "3d8fa1127a2ab4ef", - "outputs": [], - "execution_count": 1 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "`ensemble_data` will be a python dict within a dict that has a list. The outermost dict is keyed by the run number, the next dict level by the timestamp, and the list within that dict will be of all the data that was registered for the instance run.", - "id": "a47d457006cb62c0" + "id": "a47d457006cb62c0", + "metadata": {}, + "source": [ + "`ensemble_data` will be a python dict within a dict that has a list. The outermost dict is keyed by the run number, the next dict level by the timestamp, and the list within that dict will be of all the data that was registered for the instance run." + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "ensemble_data", - "id": "baab25e709153aec" + "id": "baab25e709153aec", + "metadata": {}, + "outputs": [], + "source": [ + "ensemble_data" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "For this example, we're interested in aggregating the instance data found in all the CSV files. (The JSON files contain the solutions for each instance for which there are separate jupyter notebooks, and so are outside the scope of this notebook.) So, let's create a pandas dataframe from all the CSV files.", - "id": "be9c1c201521542c" + "id": "be9c1c201521542c", + "metadata": {}, + "source": [ + "For this example, we're interested in aggregating the instance data found in all the CSV files. (The JSON files contain the solutions for each instance for which there are separate jupyter notebooks, and so are outside the scope of this notebook.) So, let's create a pandas dataframe from all the CSV files." + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "94a8c3dce6c33ae6", + "metadata": {}, + "outputs": [], "source": [ "rows = []\n", "for k in ensemble_data.keys():\n", @@ -81,57 +86,61 @@ " print(f'Reading {csv_path}')\n", " with csv_path.open('r') as csv_file:\n", " csv_reader = csv.DictReader(csv_file)\n", - " for row in csv_reader:\n", - " rows.append(row)" - ], - "id": "94a8c3dce6c33ae6" + " rows.extend(list(csv_reader))" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "df = pd.DataFrame(rows)", - "id": "3954d70afe1b1794" + "id": "3954d70afe1b1794", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(rows)" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "df", - "id": "4ddbb45fc4de7" + "id": "4ddbb45fc4de7", + "metadata": {}, + "outputs": [], + "source": [ + "df" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "1bf3f724abe263db", + "metadata": {}, + "outputs": [], "source": [ "# convert start and end times to floats to make new duration column\n", "df['start'] = df['start'].astype(float)\n", "df['end'] = df['end'].astype(float)" - ], - "id": "1bf3f724abe263db" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "df['duration'] = df['end'] - df['start']", - "id": "e1a339aacb56d29f" + "id": "e1a339aacb56d29f", + "metadata": {}, + "outputs": [], + "source": [ + "df['duration'] = df['end'] - df['start']" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "e2ec1f8dca7ad3e8", + "metadata": {}, + "outputs": [], "source": [ "# Show the final dataframe with new duration column\n", "df" - ], - "id": "e2ec1f8dca7ad3e8" + ] } ], "metadata": { diff --git a/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb b/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb index 67a7f8dd..12cee907 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb +++ b/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb @@ -1,60 +1,60 @@ { "cells": [ { - "metadata": {}, "cell_type": "markdown", + "id": "56c3997f94a366d5", + "metadata": {}, "source": [ "# Instance base notebook\n", "\n", "This notebook replicates for each ensemble instance." - ], - "id": "56c3997f94a366d5" + ] }, { + "cell_type": "code", + "execution_count": 5, + "id": "937aff95abd359fa", "metadata": { "ExecuteTime": { "end_time": "2025-12-18T18:45:35.472364Z", "start_time": "2025-12-18T18:45:35.462785Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "import json\n", - "from pathlib import Path\n", "\n", "import matplotlib.pyplot as plt" - ], - "id": "937aff95abd359fa", - "outputs": [], - "execution_count": 5 + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "fc9bfad1db6fe246", + "metadata": {}, + "outputs": [], "source": [ "# get the JSON data file generated from the instance\n", "data_files = ips_analysis_api.get_data()" - ], - "id": "fc9bfad1db6fe246" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "8f9fac74f66daa6b", + "metadata": {}, + "outputs": [], "source": [ "# echo what we got for a sanity check\n", "data_files" - ], - "id": "8f9fac74f66daa6b" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "c0883abd61aa1b18", + "metadata": {}, + "outputs": [], "source": [ "for file in data_files[0.0]:\n", " if file.endswith('_solution.json'):\n", @@ -62,44 +62,43 @@ " break\n", "else:\n", " print('JSON solution file for this ensemble is missing')" - ], - "id": "c0883abd61aa1b18" + ] }, { + "cell_type": "code", + "execution_count": 1, + "id": "b1a40da330ba6b3e", "metadata": { "ExecuteTime": { "end_time": "2025-12-18T18:40:41.874866Z", "start_time": "2025-12-18T18:40:41.864743Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "# Now read the JSON file and extract the data.\n", - "with open(solution_json_file, \"r\") as f:\n", + "with open(solution_json_file, 'r') as f:\n", " solution_data = json.load(f)\n", "\n", - "x = solution_data[\"x\"]\n", - "u = solution_data[\"u\"]\n", + "x = solution_data['x']\n", + "u = solution_data['u']\n", "\n", "# Now plot the data.\n", "plt.plot(x, u)\n", - "plt.xlabel(\"Position (x)\")\n", - "plt.ylabel(\"Temperature (u)\")\n", - "plt.title(\"Solution of 1D Heat Equation\")\n", + "plt.xlabel('Position (x)')\n", + "plt.ylabel('Temperature (u)')\n", + "plt.title('Solution of 1D Heat Equation')\n", "plt.grid(True)\n", - "plt.savefig(f\"solution.png\")" - ], - "id": "b1a40da330ba6b3e", - "outputs": [], - "execution_count": 1 + "plt.savefig('solution.png')" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "", - "id": "66002c188c5ab473" + "id": "66002c188c5ab473", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples-proposed/024-aggregated-compute-ensemble/instance_component.py b/examples-proposed/024-aggregated-compute-ensemble/instance_component.py index 341551f5..013ab10c 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/instance_component.py +++ b/examples-proposed/024-aggregated-compute-ensemble/instance_component.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Component to be stepped in instance. @@ -6,37 +5,49 @@ two are from synthetic data generated from `gen_data.py`. The latter is also generated from provenance data captured in `gen_data.py`, too. """ + from pathlib import Path -from time import time from typing import Any from ipsframework import Component -def create_cmd(instance: str, path: Path, alpha: float, L:float, T_final:float, - Nx:int, Nt:int) -> list[Any]: - """ create the command to run the external data generator +def create_cmd( + instance: str, path: Path, alpha: float, l: float, t_final: float, nx: int, nt: int +) -> list[Any]: + """create the command to run the external data generator :param instance: instance name :param path: path to data generator script directory :param alpha: thermal diffusivity - :param L: domain length - :param T_final: final time - :param Nx: number of spatial grid points - :param Nt: number of time steps + :param l: domain length + :param t_final: final time + :param nx: number of spatial grid points + :param nt: number of time steps :returns: list of command line arguments to be executed in step() """ executable = path / 'gen_data.py' - cmd = ['python3', str(executable), '--instance', instance, - '--alpha', alpha, '--L', L, '--T_final', T_final, - '--Nx', Nx, '--Nt', Nt] + cmd = [ + 'python3', + str(executable), + '--instance', + instance, + '--alpha', + alpha, + '--l', + l, + '--t_final', + t_final, + '--nx', + nx, + '--nt', + nt, + ] return cmd class InstanceComponent(Component): def step(self, timestamp: float = 0.0, **keywords): - start = time() - # ENSEMBLE_INSTANCE is a special IPS variable that contains the # string uniquely identifying this instance. Each instance will have # the `run_ensemble()` `name` argument prepended to a unique number @@ -45,37 +56,34 @@ def step(self, timestamp: float = 0.0, **keywords): self.services.info(f'{instance_id}: Start of step of instance component.') # Echo the parameters we're expecting, A, B, and C - self.services.info(f'{instance_id}: instance component parameters: ' - f'alpha={self.alpha}, L={self.L}, ' - f'T_final={self.T_final}, Nx={self.Nx}, ' - f'Nt={self.Nt}') + self.services.info( + f'{instance_id}: instance component parameters: alpha={self.alpha}, l={self.l}, t_final={self.t_final}, nx={self.nx}, nt={self.nt}' + ) - cmd = create_cmd(instance_id, Path(self.BIN_PATH), - self.alpha, self.L, self.T_final, self.Nx, self.Nt) + cmd = create_cmd( + instance_id, Path(self.BIN_PATH), self.alpha, self.l, self.t_final, self.nx, self.nt + ) working_dir = str(Path('.').absolute()) - self.services.info(f'{instance_id}: Launching executable ' - f'in {working_dir}') + self.services.info(f'{instance_id}: Launching executable in {working_dir}') run_id = None try: - cmd = ' '.join(cmd) # need one big ole string for executing tasks - run_id = self.services.launch_task(nproc=1, - working_dir=working_dir, - binary=cmd) - except Exception as e: - self.services.critical(f'{instance_id}: Unable to launch ' - f'executable in {working_dir}') + cmd = ' '.join(cmd) # need one big ole string for executing tasks + run_id = self.services.launch_task(nproc=1, working_dir=working_dir, binary=cmd) + except Exception: + self.services.critical(f'{instance_id}: Unable to launch executable in {working_dir}') return_value = self.services.wait_task(run_id) # block until done - self.services.info(f'{instance_id}: Completed MPI executable with ' - f'return value: {return_value}.') + self.services.info( + f'{instance_id}: Completed MPI executable with return value: {return_value}.' + ) # Add the generated data JSON and CSV files to the portal try: - self.services.add_analysis_data_files([f'{instance_id}_solution.json', - f'{instance_id}_stats.csv'], - replace=True) + self.services.add_analysis_data_files( + [f'{instance_id}_solution.json', f'{instance_id}_stats.csv'], replace=True + ) except Exception: print('did not add data files to portal, check logs') diff --git a/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py b/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py index 8b8f76d0..2cd89525 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py +++ b/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Driver component for instances """ @@ -6,22 +5,22 @@ from ipsframework import Component # The notebook that will be copied for each ensemble instance -SOURCE_NOTEBOOK_NAME='instance_base_notebook.ipynb' +SOURCE_NOTEBOOK_NAME = 'instance_base_notebook.ipynb' class InstanceDriver(Component): """ Instance driver component that steps the main component """ + def init(self, timestamp: float = 0.0, **keywords): self.services.stage_input_files([SOURCE_NOTEBOOK_NAME]) self.services.initialize_jupyter_notebook( - dest_notebook_name='jupyterhub_instance_notebook.ipynb', - source_notebook_path=SOURCE_NOTEBOOK_NAME, + dest_notebook_name='jupyterhub_instance_notebook.ipynb', + source_notebook_path=SOURCE_NOTEBOOK_NAME, ) - def step(self, timestamp: float = 0.0, **keywords): instance_component = self.services.get_port('WORKER') diff --git a/examples-proposed/dask/dask_worker.py b/examples-proposed/dask/dask_worker.py index a87b92d4..66c982ca 100644 --- a/examples-proposed/dask/dask_worker.py +++ b/examples-proposed/dask/dask_worker.py @@ -4,8 +4,8 @@ from ipsframework import Component -def myFun(*args): - print(f'myFun({args[0]})') +def my_fun(*args): + print(f'my_fun({args[0]})') sleep(float(args[0])) return 0 @@ -17,15 +17,15 @@ def step(self, timestamp=0.0): duration = 0.5 self.services.add_task('pool', 'binary', 1, cwd, self.EXECUTABLE, duration) - self.services.add_task('pool', 'function', 1, cwd, myFun, duration) - self.services.add_task('pool', 'method', 1, cwd, copy.copy(self).myMethod, duration) + self.services.add_task('pool', 'function', 1, cwd, my_fun, duration) + self.services.add_task('pool', 'method', 1, cwd, copy.copy(self).my_method, duration) ret_val = self.services.submit_tasks('pool', use_dask=True, dask_nodes=1) print('ret_val =', ret_val) exit_status = self.services.get_finished_tasks('pool') print('exit_status = ', exit_status) - def myMethod(self, *args): - print(f'myMethod({args[0]})') + def my_method(self, *args): + print(f'my_method({args[0]})') sleep(float(args[0])) return 0 diff --git a/examples-proposed/dask/simulation_log.json b/examples-proposed/dask/simulation_log.json index 1d7b88fd..07d66c32 100644 --- a/examples-proposed/dask/simulation_log.json +++ b/examples-proposed/dask/simulation_log.json @@ -8,7 +8,7 @@ "code": "DASK_WORKER__DaskWorker", "eventtype": "IPS_LAUNCH_DASK_TASK", "walltime": "2.33", - "comment": "task_name = function, Target = myFun(0.5)", + "comment": "task_name = function, Target = my_fun(0.5)", } { "code": "DASK_WORKER__DaskWorker", diff --git a/ipsframework/__init__.py b/ipsframework/__init__.py index f248cfc4..7f52cc31 100644 --- a/ipsframework/__init__.py +++ b/ipsframework/__init__.py @@ -1,14 +1,24 @@ """IPS Framework""" from .component import Component -from .configurationManager import ConfigurationManager -from .dataManager import DataManager +from .configuration_manager import ConfigurationManager +from .data_manager import DataManager from .ips import Framework -from .resourceManager import ResourceManager +from .resource_manager import ResourceManager from .services import ServicesProxy, Task, TaskPool -from .taskManager import TaskManager +from .task_manager import TaskManager -__all__ = ['Component', 'ConfigurationManager', 'DataManager', 'Framework', 'ResourceManager', 'ServicesProxy', 'Task', 'TaskManager', 'TaskPool'] +__all__ = [ + 'Component', + 'ConfigurationManager', + 'DataManager', + 'Framework', + 'ResourceManager', + 'ServicesProxy', + 'Task', + 'TaskManager', + 'TaskPool', +] from . import _version diff --git a/ipsframework/_internal/bridges/local_logging_bridge.py b/ipsframework/_internal/bridges/local_logging_bridge.py index 09f8f226..2b1f038c 100644 --- a/ipsframework/_internal/bridges/local_logging_bridge.py +++ b/ipsframework/_internal/bridges/local_logging_bridge.py @@ -5,7 +5,7 @@ import json import os import time -from typing import TYPE_CHECKING, Any, Literal, Union +from typing import TYPE_CHECKING, Any, Literal from ipsframework import Component, ipsutil from ipsframework.cca_es_spec import Event @@ -24,12 +24,12 @@ def __init__(self): self.counter = 0 self.monitor_file_prefix = '' """The name of the file, minus the extension ('.html', '.jsonl', etc.). - + If this is empty, you must either check to see if you can create the file, or you should assume that you can't create the file. """ - self.portal_runid: Union[str, None] = None + self.portal_runid: str | None = None """Portal RunID, set by component which publishes the IPS_START event. Only used for logging here.""" - self.parent_portal_runid: Union[str, None] = None + self.parent_portal_runid: str | None = None """Parent portal RunID, derived from locally determined portal RunID. Should explicitly be None (not empty string) if not set. Only used for logging.""" self.sim_name = '' self.sim_root = '' @@ -79,11 +79,15 @@ def init(self, timestamp=0.0, **keywords): try: self.html_dir = self.services.get_config_param('USER_W3_DIR', silent=True) or '' except Exception: - self.services.warning('Missing USER_W3_DIR configuration - disabling web-visible logging') + self.services.warning( + 'Missing USER_W3_DIR configuration - disabling web-visible logging' + ) self.write_to_htmldir = False else: if self.html_dir.strip() == '': - self.services.warning('Empty USER_W3_DIR configuration - disabling web-visible logging') + self.services.warning( + 'Empty USER_W3_DIR configuration - disabling web-visible logging' + ) self.write_to_htmldir = False else: try: @@ -91,7 +95,9 @@ def init(self, timestamp=0.0, **keywords): except FileExistsError: pass except Exception: - self.services.warning('Unable to create HTML directory - disabling web-visible logging') + self.services.warning( + 'Unable to create HTML directory - disabling web-visible logging' + ) self.write_to_htmldir = False def step(self, timestamp=0.0, **keywords): @@ -110,11 +116,11 @@ def finalize(self, timestamp=0.0, **keywords): except Exception: pass - def process_event(self, topicName: str, theEvent: Event): + def process_event(self, topic_name: str, the_event: Event): """ - Process a single event *theEvent* on topic *topicName*. + Process a single event *the_event* on topic *topic_name*. """ - event_body = theEvent.getBody() + event_body = the_event.get_body() sim_name = event_body['sim_name'] portal_data = event_body['portal_data'] try: @@ -154,7 +160,9 @@ def process_event(self, topicName: str, theEvent: Event): portal_data['seqnum'] = sim_data.counter if 'trace' in portal_data: - portal_data['trace']['traceId'] = hashlib.md5(sim_data.portal_runid.encode()).hexdigest() + portal_data['trace']['traceId'] = hashlib.md5( + sim_data.portal_runid.encode() + ).hexdigest() self.send_event(sim_data, portal_data) @@ -172,7 +180,9 @@ def init_simulation(self, sim_name: str, sim_root: str, portal_runid: str): *sim_root* so the portal can set up corresponding structures to manage data from the sim. """ - self.services.debug('Initializing simulation using BasicBridge: %s -- %s ', sim_name, sim_root) + self.services.debug( + 'Initializing simulation using BasicBridge: %s -- %s ', sim_name, sim_root + ) sim_data = SimulationData() sim_data.sim_name = sim_name @@ -191,15 +201,21 @@ def init_simulation(self, sim_name: str, sim_root: str, portal_runid: str): try: os.makedirs(sim_log_dir, exist_ok=True) except OSError as oserr: - self.services.exception('Error creating Simulation Log directory %s : %d %s' % (sim_log_dir, oserr.errno, oserr.strerror)) + self.services.exception( + 'Error creating Simulation Log directory %s : %d %s' + % (sim_log_dir, oserr.errno, oserr.strerror) + ) raise sim_data.monitor_file_prefix = os.path.join(sim_log_dir, sim_data.portal_runid) eventlog_fname = f'{sim_data.monitor_file_prefix}.eventlog' try: sim_data.monitor_file = open(eventlog_fname, 'wb', 0) - except IOError as oserr: - self.services.error('Error opening file %s: error(%s): %s' % (eventlog_fname, oserr.errno, oserr.strerror)) + except OSError as oserr: + self.services.error( + 'Error opening file %s: error(%s): %s' + % (eventlog_fname, oserr.errno, oserr.strerror) + ) self.services.error('Using /dev/null instead') sim_data.monitor_file_prefix = '' sim_data.monitor_file = open('/dev/null', 'w') @@ -223,7 +239,7 @@ def send_event(self, sim_data: SimulationData, event_data: dict[str, Any]): """ Send contents of *event_data* and *sim_data* to portal. """ - timestamp = ipsutil.getTimeString() + timestamp = ipsutil.get_time_string() buf = '%8d %s ' % (sim_data.counter, timestamp) for k, v in event_data.items(): if len(str(v).strip()) == 0: @@ -240,7 +256,10 @@ def send_event(self, sim_data: SimulationData, event_data: dict[str, Any]): sim_data.json_monitor_file.write('%s\n' % buf) freq = self.dump_freq - if ((self.counter % freq == 0) and (time.time() - self.last_dump_time > self.min_dump_interval)) or (event_data['eventtype'] == 'IPS_END'): + if ( + (self.counter % freq == 0) + and (time.time() - self.last_dump_time > self.min_dump_interval) + ) or (event_data['eventtype'] == 'IPS_END'): self.last_dump_time = time.time() if sim_data.monitor_file_prefix: html_filename = f'{sim_data.monitor_file_prefix}.html' @@ -251,5 +270,7 @@ def send_event(self, sim_data: SimulationData, event_data: dict[str, Any]): try: open(html_file, 'w').writelines(html_page) except Exception: - self.services.exception('Error writing html file into USER_W3_DIR directory') + self.services.exception( + 'Error writing html file into USER_W3_DIR directory' + ) self.write_to_htmldir = False diff --git a/ipsframework/_internal/bridges/portal_bridge.py b/ipsframework/_internal/bridges/portal_bridge.py index 3ebe4df0..8a75a013 100644 --- a/ipsframework/_internal/bridges/portal_bridge.py +++ b/ipsframework/_internal/bridges/portal_bridge.py @@ -8,11 +8,12 @@ import os import tarfile import time +from collections.abc import Callable from multiprocessing import Event, Pipe, Process from multiprocessing.connection import Connection from multiprocessing.synchronize import Event as EventType from pathlib import Path -from typing import Any, Callable, Literal, Union +from typing import Any, Literal from urllib3 import PoolManager from urllib3.exceptions import MaxRetryError @@ -66,7 +67,9 @@ def send_post(conn: Connection, stop: EventType, url: str): def send_jupyter_notebook(conn: Connection, stop: EventType, url: str, api_key: str, username: str): fail_count = 0 - http = PoolManager(retries=Urllib3Retry(total=MAX_RETRIES, backoff_factor=1, respect_retry_after_header=True)) + http = PoolManager( + retries=Urllib3Retry(total=MAX_RETRIES, backoff_factor=1, respect_retry_after_header=True) + ) while True: if conn.poll(0.1): @@ -92,22 +95,42 @@ def send_jupyter_notebook(conn: Connection, stop: EventType, url: str, api_key: ) except MaxRetryError as e: fail_count += 1 - conn.send((NOTEBOOK_MESSAGE_TYPE, next_val['component_id'], 999, f'Max retry error: {e}')) + conn.send( + (NOTEBOOK_MESSAGE_TYPE, next_val['component_id'], 999, f'Max retry error: {e}') + ) else: - conn.send((NOTEBOOK_MESSAGE_TYPE, next_val['component_id'], resp.status, resp.data.decode())) + conn.send( + ( + NOTEBOOK_MESSAGE_TYPE, + next_val['component_id'], + resp.status, + resp.data.decode(), + ) + ) fail_count = 0 if fail_count >= MAX_RETRIES: - conn.send((NOTEBOOK_MESSAGE_TYPE, next_val['component_id'], -1, 'Too many consecutive failed connections')) + conn.send( + ( + NOTEBOOK_MESSAGE_TYPE, + next_val['component_id'], + -1, + 'Too many consecutive failed connections', + ) + ) break elif stop.is_set(): break -def send_jupyter_notebook_data(conn: Connection, stop: EventType, url: str, api_key: str, username: str): +def send_jupyter_notebook_data( + conn: Connection, stop: EventType, url: str, api_key: str, username: str +): fail_count = 0 - http = PoolManager(retries=Urllib3Retry(total=MAX_RETRIES, backoff_factor=1, respect_retry_after_header=True)) + http = PoolManager( + retries=Urllib3Retry(total=MAX_RETRIES, backoff_factor=1, respect_retry_after_header=True) + ) while True: if conn.poll(0.1): @@ -143,22 +166,37 @@ def send_jupyter_notebook_data(conn: Connection, stop: EventType, url: str, api_ ) except (MaxRetryError, OSError) as e: fail_count += 1 - conn.send((DATA_MESSAGE_TYPE, next_val['component_id'], 999, f'Max retry error: {e}')) + conn.send( + (DATA_MESSAGE_TYPE, next_val['component_id'], 999, f'Max retry error: {e}') + ) else: - conn.send((DATA_MESSAGE_TYPE, next_val['component_id'], resp.status, resp.data.decode())) + conn.send( + (DATA_MESSAGE_TYPE, next_val['component_id'], resp.status, resp.data.decode()) + ) fail_count = 0 if fail_count >= MAX_RETRIES: - conn.send((DATA_MESSAGE_TYPE, next_val['component_id'], -1, 'Too many consecutive failed connections')) + conn.send( + ( + DATA_MESSAGE_TYPE, + next_val['component_id'], + -1, + 'Too many consecutive failed connections', + ) + ) break elif stop.is_set(): break -def send_ensemble_variables(conn: Connection, stop: EventType, url: str, api_key: str, username: str): +def send_ensemble_variables( + conn: Connection, stop: EventType, url: str, api_key: str, username: str +): fail_count = 0 - http = PoolManager(retries=Urllib3Retry(total=MAX_RETRIES, backoff_factor=1, respect_retry_after_header=True)) + http = PoolManager( + retries=Urllib3Retry(total=MAX_RETRIES, backoff_factor=1, respect_retry_after_header=True) + ) while True: if conn.poll(0.1): @@ -189,13 +227,29 @@ def send_ensemble_variables(conn: Connection, stop: EventType, url: str, api_key ) except (MaxRetryError, OSError) as e: fail_count += 1 - conn.send((ENSEMBLE_MESSAGE_TYPE, next_val['component_id'], 999, f'Max retry error: {e}')) + conn.send( + (ENSEMBLE_MESSAGE_TYPE, next_val['component_id'], 999, f'Max retry error: {e}') + ) else: - conn.send((ENSEMBLE_MESSAGE_TYPE, next_val['component_id'], resp.status, resp.data.decode())) + conn.send( + ( + ENSEMBLE_MESSAGE_TYPE, + next_val['component_id'], + resp.status, + resp.data.decode(), + ) + ) fail_count = 0 if fail_count >= MAX_RETRIES: - conn.send((ENSEMBLE_MESSAGE_TYPE, next_val['component_id'], -1, 'Too many consecutive failed connections')) + conn.send( + ( + ENSEMBLE_MESSAGE_TYPE, + next_val['component_id'], + -1, + 'Too many consecutive failed connections', + ) + ) break elif stop.is_set(): break @@ -210,7 +264,9 @@ def __init__(self, target: Callable, *args): """ self.parent_conn, self.child_conn = Pipe() self.childProcessStop = Event() - self.childProcess = Process(target=target, args=(self.child_conn, self.childProcessStop, *args)) + self.childProcess = Process( + target=target, args=(self.child_conn, self.childProcessStop, *args) + ) self.childProcess.start() @@ -221,9 +277,9 @@ class PortalSimulationData: def __init__(self): self.counter = 0 - self.portal_runid: Union[str, None] = None + self.portal_runid: str | None = None """Locally determined portal runid, also sent to the portal.""" - self.parent_portal_runid: Union[str, None] = None + self.parent_portal_runid: str | None = None """Parent portal runid, derived from locally determined portal runid. Should explicitly be None (not empty string) if not set.""" self.sim_name = '' self.sim_root = '' @@ -317,11 +373,11 @@ def terminate(self, status: Literal[0, 1]): ### SUBSCRIPTION CHANNELS (public) ### - def process_event(self, topicName, theEvent): + def process_event(self, topic_name, the_event): """ - Process a single event *theEvent* on topic *topicName*. + Process a single event *the_event* on topic *topic_name*. """ - event_body = theEvent.getBody() + event_body = the_event.get_body() sim_name = event_body['sim_name'] portal_data = event_body['portal_data'] try: @@ -381,7 +437,9 @@ def process_event(self, topicName, theEvent): portal_data['seqnum'] = sim_data.counter if 'trace' in portal_data: - portal_data['trace']['traceId'] = hashlib.md5(sim_data.portal_runid.encode()).hexdigest() + portal_data['trace']['traceId'] = hashlib.md5( + sim_data.portal_runid.encode() + ).hexdigest() if self.portal_url: polling_timeout = 0.0 @@ -454,7 +512,11 @@ def _check_send_post_responses(self, polling_timeout: float = 0.0): def _check_url_manager_responses(self): """poll all data API checks""" - for manager in [self.url_manager_jupyter_notebook, self.url_manager_jupyter_data, self.url_manager_ensemble_uploads]: + for manager in [ + self.url_manager_jupyter_notebook, + self.url_manager_jupyter_data, + self.url_manager_ensemble_uploads, + ]: if manager is None: continue while manager.parent_conn.poll(): @@ -472,7 +534,11 @@ def _check_url_manager_responses(self): else: self.services.debug('Portal Response: %d %s', code, msg) if msg_type == ENSEMBLE_MESSAGE_TYPE: - self.services.publish(f'_IPS_{component_id}', '_IPS_PORTAL_UPLOAD_ENSEMBLE_PARAMS_SUCCESS', 'true') + self.services.publish( + f'_IPS_{component_id}', + '_IPS_PORTAL_UPLOAD_ENSEMBLE_PARAMS_SUCCESS', + 'true', + ) def _send_jupyter_notebook(self, sim_data: PortalSimulationData, event_data): if self.portal_url and self.portal_api_key: @@ -540,7 +606,9 @@ def init_simulation(self, sim_name: str, sim_root: str, portal_runid: str): sim_data.portal_runid = portal_runid try: - self.services.set_config_param('PORTAL_RUNID', sim_data.portal_runid, target_sim_name=sim_name) + self.services.set_config_param( + 'PORTAL_RUNID', sim_data.portal_runid, target_sim_name=sim_name + ) except Exception: self.services.error('Simulation %s is not accessible', sim_name) return diff --git a/ipsframework/cca_es_spec.py b/ipsframework/cca_es_spec.py index 23b8039b..9ff0cb9b 100644 --- a/ipsframework/cca_es_spec.py +++ b/ipsframework/cca_es_spec.py @@ -14,7 +14,7 @@ _proxy = None -class EventServiceException(Exception): +class EventServiceError(Exception): """ Exception class for the event service. """ @@ -32,33 +32,33 @@ class PublisherEventService: Interface to topics for publishers. """ - def getTopic(self, topicName): + def get_topic(self, topic_name): """ """ - return _proxy.getTopic(topicName) + return _proxy.get_topic(topic_name) - def existsTopic(self, topicName): - return _proxy.existsTopic(topicName) + def exists_topic(self, topic_name): + return _proxy.exists_topic(topic_name) class SubscriberEventService: def __init__(self): - self.subscriberid = _proxy.registerSubscriber() + self.subscriberid = _proxy.register_subscriber() - def getSubscription(self, subscriptionName): + def get_subscription(self, subscription_name): """ A Subscription object can be safely returned from here without screwing up automatic object tracking for cleaning up out-of-scope subscriptions. A framework/component subscriber uses this Subscription object to talk to the event service. """ - _proxy.getSubscription(self.subscriberid, subscriptionName) - return Subscription(self.subscriberid, subscriptionName) + _proxy.get_subscription(self.subscriberid, subscription_name) + return Subscription(self.subscriberid, subscription_name) - def processEvents(self): - _proxy.processEvents(self.subscriberid) + def process_events(self): + _proxy.process_events(self.subscriberid) def __del__(self): - _proxy.unregisterSubscriber(self.subscriberid) + _proxy.unregister_subscriber(self.subscriberid) class Event: @@ -66,10 +66,10 @@ def __init__(self, header, body): self.header = deepcopy(header) self.body = deepcopy(body) - def getHeader(self): + def get_header(self): return self.header - def getBody(self): + def get_body(self): return self.body def __str__(self) -> str: @@ -78,11 +78,11 @@ def __str__(self) -> str: class EventListener: def __init__(self): - self.listenerid = _proxy.createListener() + self.listenerid = _proxy.create_listener() - def processEvent(self, topicName, theEvent): + def process_event(self, topic_name, the_event): """ - A listener implements the processEvent method to respond to an event, + A listener implements the process_event method to respond to an event, thereby overriding the below invocation. Ideally, it should be an abstract method, but currently serves to check the correct operation of the event service. @@ -90,32 +90,38 @@ def processEvent(self, topicName, theEvent): class Topic: - def __init__(self, topicName): - self.topicName = topicName + def __init__(self, topic_name): + self.topic_name = topic_name - def getTopicName(self): - return self.topicName + def get_topic_name(self): + return self.topic_name - def sendEvent(self, eventName, eventBody): - _proxy.sendEvent(self.topicName, eventName, eventBody) + def send_event(self, event_name, event_body): + _proxy.send_event(self.topic_name, event_name, event_body) class Subscription: - def __init__(self, subscriberid, subscriptionName): + def __init__(self, subscriberid, subscription_name): self.subscriberid = subscriberid - self.subscriptionName = subscriptionName + self.subscription_name = subscription_name - def registerEventListener(self, listenerKey, theListener): - _proxy.registerEventListener(self.subscriberid, self.subscriptionName, listenerKey, theListener.listenerid, theListener) + def register_event_listener(self, listener_key, the_listener): + _proxy.register_event_listener( + self.subscriberid, + self.subscription_name, + listener_key, + the_listener.listenerid, + the_listener, + ) - def unregisterEventListener(self, listenerKey): - _proxy.unregisterEventListener(self.subscriberid, self.subscriptionName, listenerKey) + def unregister_event_listener(self, listener_key): + _proxy.unregister_event_listener(self.subscriberid, self.subscription_name, listener_key) - def getSubscriptionName(self): - return self.subscriptionName + def get_subscription_name(self): + return self.subscription_name def __del__(self): - _proxy.removeSubscription(self.subscriberid, self.subscriptionName) + _proxy.remove_subscription(self.subscriberid, self.subscription_name) """ Initialize the proxy """ @@ -130,6 +136,5 @@ def initialize_event_service(service): _proxy = EventServiceCmpProxy(service) -# pylint: disable=wrong-import-position -from .eventService import EventService # noqa: E402 -from .eventServiceProxy import EventServiceCmpProxy, EventServiceFwkProxy # noqa: E402 +from .event_service import EventService +from .event_service_proxy import EventServiceCmpProxy, EventServiceFwkProxy diff --git a/ipsframework/component.py b/ipsframework/component.py index 45ea5213..a73beb2d 100644 --- a/ipsframework/component.py +++ b/ipsframework/component.py @@ -8,9 +8,9 @@ import weakref from copy import copy from multiprocessing import Queue -from typing import TYPE_CHECKING, Any, Dict, Literal +from typing import TYPE_CHECKING, Any, Literal -from .componentRegistry import ComponentID +from .component_registry import ComponentID from .messages import Message, MethodResultMessage if TYPE_CHECKING: @@ -29,7 +29,7 @@ class Component: :type config: dict """ - def __init__(self, services, config: Dict[str, Any]): + def __init__(self, services, config: dict[str, Any]): """ Set up config values and reference to services. """ @@ -59,7 +59,9 @@ def __copy__(self): setattr(result, k, copy(v)) return result - def __initialize__(self, component_id: ComponentID, invocation_q: Queue, start_time: float = 0.0): + def __initialize__( + self, component_id: ComponentID, invocation_q: Queue, start_time: float = 0.0 + ): """ Establish connection to *invocation_q*. """ @@ -127,7 +129,10 @@ def __run__(self): self.services._init_event_service() # the topic prefix must start with '_IPS_' to be a reserved topic - self.services.subscribe(f'_IPS_{self.__component_id.get_serialization()}', self.services._component_id_subscription_callback) + self.services.subscribe( + f'_IPS_{self.__component_id.get_serialization()}', + self.services._component_id_subscription_callback, + ) while True: msg = self.__invocation_q.get() @@ -141,15 +146,21 @@ def __run__(self): if keywords: formatted_args += [' %s=' % k + str(v) for (k, v) in keywords.items()] - self.services.debug('Calling method ' + self.method_name + '(' + ' ,'.join(formatted_args) + ')') + self.services.debug( + 'Calling method ' + self.method_name + '(' + ' ,'.join(formatted_args) + ')' + ) try: method = getattr(self, self.method_name) retval = method(*self.args, **keywords) except Exception as e: self.services.exception('Uncaught Exception in component method.') - response_msg = MethodResultMessage(self.component_id, sender_id, self.call_id, Message.FAILURE, e) + response_msg = MethodResultMessage( + self.component_id, sender_id, self.call_id, Message.FAILURE, e + ) else: - response_msg = MethodResultMessage(self.component_id, sender_id, self.call_id, Message.SUCCESS, retval) + response_msg = MethodResultMessage( + self.component_id, sender_id, self.call_id, Message.SUCCESS, retval + ) self.services.fwk_in_q.put(response_msg) @property diff --git a/ipsframework/componentRegistry.py b/ipsframework/component_registry.py similarity index 85% rename from ipsframework/componentRegistry.py rename to ipsframework/component_registry.py index e4ba066d..32306831 100644 --- a/ipsframework/componentRegistry.py +++ b/ipsframework/component_registry.py @@ -12,7 +12,7 @@ class SingletonMeta(type): def __call__(cls, *args, **kwargs): if cls not in cls.__instances: - cls.__instances[cls] = super(SingletonMeta, cls).__call__(*args, **kwargs) + cls.__instances[cls] = super().__call__(*args, **kwargs) return cls.__instances[cls] @@ -66,6 +66,9 @@ def __repr__(self): def __eq__(self, other): return str(self) == str(other) + def __hash__(self): + return hash(str(self)) + def get_instance_name(self): """ Return instance name of component id. @@ -121,10 +124,16 @@ def get_component_ids(self, sim_name): """ Return all of the component ids associated with sim *sim_name* """ - ids = [ComponentID.deserialize(i) for i in self.registry if ComponentID.deserialize(i).get_sim_name() == sim_name] + ids = [ + ComponentID.deserialize(i) + for i in self.registry + if ComponentID.deserialize(i).get_sim_name() == sim_name + ] return ids - def addEntry(self, component_id, svc_response_q, invocation_q, component_ref, services, config): + def add_entry( + self, component_id, svc_response_q, invocation_q, component_ref, services, config + ): """ Create a component registry entry for *component_id* and its associated queues, component ref, services and configuration @@ -135,20 +144,24 @@ def addEntry(self, component_id, svc_response_q, invocation_q, component_ref, se try: self.registry[key] = value except KeyError as e: - print('Error creating component registry entry for ', key, ' : ', str(e), file=sys.stderr) + print( + 'Error creating component registry entry for ', key, ' : ', str(e), file=sys.stderr + ) raise e - def removeEntry(self, component_id): + def remove_entry(self, component_id): key = component_id.get_serialization() try: del self.registry[key] except KeyError as e: - print('Error removing component registry entry for ', key, ' : ', str(e), file=sys.stderr) + print( + 'Error removing component registry entry for ', key, ' : ', str(e), file=sys.stderr + ) raise # SIMYAN: this was added to provide an easy way to use the component # registry to get a registry entry - def getEntry(self, component_id): + def get_entry(self, component_id): """ Return a registry entry. """ @@ -160,7 +173,7 @@ def getEntry(self, component_id): raise return entry - def getComponentArtifact(self, component_id, artifact): + def get_component_artifact(self, component_id, artifact): """ Return value of *artifact* in *component_id*'s registry entry. """ @@ -178,7 +191,7 @@ def getComponentArtifact(self, component_id, artifact): raise return value - def setComponentArtifact(self, component_id, artifact, value): + def set_component_artifact(self, component_id, artifact, value): """ Set the value of *artifact* in *component_id*'s registry entry to *value*. diff --git a/ipsframework/configurationManager.py b/ipsframework/configuration_manager.py similarity index 85% rename from ipsframework/configurationManager.py rename to ipsframework/configuration_manager.py index e7bd6e39..4b8276b2 100644 --- a/ipsframework/configurationManager.py +++ b/ipsframework/configuration_manager.py @@ -10,13 +10,14 @@ import tempfile import time import uuid +from collections.abc import Iterable from multiprocessing import Process, Queue, set_start_method -from typing import Any, Iterable, Optional, Union +from typing import Any from configobj import ConfigObj -from . import ipsLogging -from .componentRegistry import ComponentID, ComponentRegistry +from .component_registry import ComponentID, ComponentRegistry +from .ips_logging import IpsLogger from .services import ServicesProxy # Try using fork for starting subprocesses, this is the default on @@ -44,7 +45,7 @@ class SimulationData: entry in the configurationManager class """ - def __init__(self, sim_name: str, start_time: Optional[float] = None) -> None: + def __init__(self, sim_name: str, start_time: float | None = None) -> None: self.start_time = start_time if start_time else time.time() self.sim_name = sim_name self.portal_sim_name = None @@ -61,7 +62,12 @@ def __init__(self, sim_name: str, start_time: Optional[float] = None) -> None: self.component_process = None self.process_list = [] - def __init__(self, fwk: Any, config_file_list: list[Union[str, os.PathLike[str]]], platform_file_name: Union[str, os.PathLike[str]]) -> None: + def __init__( + self, + fwk: Any, + config_file_list: list[str | os.PathLike[str]], + platform_file_name: str | os.PathLike[str], + ) -> None: """ Initialize the values to be used by the configuration manager. Also specified are the required fields of the simulation configuration @@ -83,7 +89,15 @@ def __init__(self, fwk: Any, config_file_list: list[Union[str, os.PathLike[str]] # in the component-generic.conf file, which allows you to point to a # directory that contains physics and other binaries on a global level # i.e. removing the requirement that it be specified for each component - self.required_fields = {'CLASS', 'SUB_CLASS', 'NAME', 'SCRIPT', 'INPUT_FILES', 'OUTPUT_FILES', 'NPROC'} + self.required_fields = { + 'CLASS', + 'SUB_CLASS', + 'NAME', + 'SCRIPT', + 'INPUT_FILES', + 'OUTPUT_FILES', + 'NPROC', + } self.config_file_list = [] self.sim_name_list = None self.sim_root_list = None @@ -117,18 +131,30 @@ def __getattr__(self, attr: str) -> Any: self.platform_file = os.path.abspath(platform_file_name) self.platform_conf = {} loc_keys = [] - mach_keys = ['MPIRUN', 'NODE_DETECTION', 'CORES_PER_NODE', 'SOCKETS_PER_NODE', 'NODE_ALLOCATION_MODE'] + mach_keys = [ + 'MPIRUN', + 'NODE_DETECTION', + 'CORES_PER_NODE', + 'SOCKETS_PER_NODE', + 'NODE_ALLOCATION_MODE', + ] prov_keys = ['HOST'] self.platform_keywords = loc_keys + mach_keys + prov_keys - self.service_methods = ['get_port', 'get_config_parameter', 'set_config_parameter', 'get_time_loop', 'create_simulation'] + self.service_methods = [ + 'get_port', + 'get_config_parameter', + 'set_config_parameter', + 'get_time_loop', + 'create_simulation', + ] self.fwk.register_service_handler(self.service_methods, self.process_service_request) self.sim_map = {} self.finished_sim_map = {} self.fwk_sim_name = None # "Fake" simconf for framework components self.fwk_components = [] # List of framework specific components self.myTopic = None - self.log_daemon = ipsLogging.ipsLogger(self.log_dynamic_sim_queue) + self.log_daemon = IpsLogger(self.log_dynamic_sim_queue) self.log_process = None # CM initialize @@ -138,7 +164,7 @@ def initialize(self, data_mgr: Any, resource_mgr: Any, task_mgr: Any) -> None: :py:obj:`ConfigObj` module. Create and initialize simulation(s) and their components, framework components and loggers. """ - self.event_mgr = None # eventManager(self) + self.event_mgr = None # EventManager(self) self.data_mgr = data_mgr self.resource_mgr = resource_mgr self.task_mgr = task_mgr @@ -157,8 +183,10 @@ def initialize(self, data_mgr: Any, resource_mgr: Any, task_mgr: Any) -> None: """ # parse file try: - self.platform_conf = ConfigObj(self.platform_file, interpolation='template', file_error=True) - except (IOError, SyntaxError): + self.platform_conf = ConfigObj( + self.platform_file, interpolation='template', file_error=True + ) + except (OSError, SyntaxError): self.fwk.exception('Error opening config file: %s', self.platform_file) raise # get mandatory values @@ -205,10 +233,16 @@ def initialize(self, data_mgr: Any, resource_mgr: Any, task_mgr: Any) -> None: try: node_alloc_mode = self.platform_conf['NODE_ALLOCATION_MODE'].upper() if node_alloc_mode not in ['EXCLUSIVE', 'SHARED']: - self.fwk.error("bad value for NODE_ALLOCATION_MODE. expected 'EXCLUSIVE' or 'SHARED'.") - raise ValueError("bad value for NODE_ALLOCATION_MODE. expected 'EXCLUSIVE' or 'SHARED'.") + self.fwk.error( + "bad value for NODE_ALLOCATION_MODE. expected 'EXCLUSIVE' or 'SHARED'." + ) + raise ValueError( + "bad value for NODE_ALLOCATION_MODE. expected 'EXCLUSIVE' or 'SHARED'." + ) except Exception: - self.fwk.exception("missing value or bad type for NODE_ALLOCATION_MODE. expected 'EXCLUSIVE' or 'SHARED'.") + self.fwk.exception( + "missing value or bad type for NODE_ALLOCATION_MODE. expected 'EXCLUSIVE' or 'SHARED'." + ) raise uan_val = self.platform_conf.get('USE_ACCURATE_NODES', 'ON').upper() @@ -253,11 +287,13 @@ def initialize(self, data_mgr: Any, resource_mgr: Any, task_mgr: Any) -> None: if 'PORTAL_API_KEY' in conf: self.platform_conf['_IPS_PORTAL_API_KEY'] = conf['PORTAL_API_KEY'] - except (IOError, SyntaxError): + except (OSError, SyntaxError): self.fwk.exception('Error opening config file %s: ', conf_file) raise except Exception: - self.fwk.exception('Error(s) during parsing of supplied config file %s: ', conf_file) + self.fwk.exception( + 'Error(s) during parsing of supplied config file %s: ', conf_file + ) raise try: @@ -339,9 +375,9 @@ def _initialize_fwk_components(self) -> None: runspace_conf = {} runspace_conf['CLASS'] = 'FWK' runspace_conf['SUB_CLASS'] = 'COMP' - runspace_conf['NAME'] = 'runspaceInitComponent' + runspace_conf['NAME'] = 'RunspaceInitComponent' runspace_conf['SCRIPT'] = '' - runspace_conf['MODULE'] = 'ipsframework.runspaceInitComponent' + runspace_conf['MODULE'] = 'ipsframework.runspace_init_component' runspace_conf['INPUT_DIR'] = '/dev/null' runspace_conf['INPUT_FILES'] = '' runspace_conf['IPS_CONFFILE_DIR'] = '' @@ -353,7 +389,9 @@ def _initialize_fwk_components(self) -> None: if self.fwk.log_level == logging.DEBUG: runspace_conf['LOG_LEVEL'] = 'DEBUG' - runspace_component_id = self._create_component(runspace_conf, self.sim_map[self.fwk_sim_name]) + runspace_component_id = self._create_component( + runspace_conf, self.sim_map[self.fwk_sim_name] + ) self.fwk_components.append(runspace_component_id) # SIMYAN: set up The Portal bridge, allowing for an absence of a portal @@ -361,7 +399,10 @@ def _initialize_fwk_components(self) -> None: # Users must set USE_PORTAL and include a PORTAL_URL in order to enable the portal if 'PORTAL_URL' not in self.sim_map[self.fwk_sim_name].sim_conf: use_portal = False - elif 'USE_PORTAL' in self.sim_map[self.fwk_sim_name].sim_conf and 'PORTAL_URL' in self.sim_map[self.fwk_sim_name].sim_conf: + elif ( + 'USE_PORTAL' in self.sim_map[self.fwk_sim_name].sim_conf + and 'PORTAL_URL' in self.sim_map[self.fwk_sim_name].sim_conf + ): use_portal = self.sim_map[self.fwk_sim_name].sim_conf['USE_PORTAL'] if isinstance(use_portal, str) and use_portal.lower().strip() == 'true': use_portal = True @@ -376,7 +417,9 @@ def _initialize_fwk_components(self) -> None: def _config_portal(config: dict[str, Any]) -> None: config['PORTAL_URL'] = self.get_platform_parameter('PORTAL_URL', silent=True) if config['PORTAL_URL']: - bridge_conf['_IPS_PORTAL_API_KEY'] = self.get_platform_parameter('_IPS_PORTAL_API_KEY', silent=True) + bridge_conf['_IPS_PORTAL_API_KEY'] = self.get_platform_parameter( + '_IPS_PORTAL_API_KEY', silent=True + ) fwk_components.append(('portal_bridge', 'PortalBridge', _config_portal)) @@ -387,7 +430,9 @@ def _config_portal(config: dict[str, Any]) -> None: bridge_conf['NAME'] = fwk_comp[1] if 'FWK_COMPS_PATH' in self.sim_map[self.fwk_sim_name].sim_conf: bridge_conf['BIN_PATH'] = self.sim_map[self.fwk_sim_name].sim_conf['FWK_COMPS_PATH'] - bridge_conf['SCRIPT'] = os.path.join(bridge_conf['BIN_PATH'], '_internal', 'bridges', f'{fwk_comp[0]}.py') + bridge_conf['SCRIPT'] = os.path.join( + bridge_conf['BIN_PATH'], '_internal', 'bridges', f'{fwk_comp[0]}.py' + ) else: bridge_conf['SCRIPT'] = '' bridge_conf['MODULE'] = f'ipsframework._internal.bridges.{fwk_comp[0]}' @@ -411,7 +456,7 @@ def _config_portal(config: dict[str, Any]) -> None: component_id = self._create_component(bridge_conf, self.sim_map[self.fwk_sim_name]) self.fwk_components.append(component_id) - def _initialize_sim(self, sim_data: "ConfigurationManager.SimulationData") -> None: + def _initialize_sim(self, sim_data: 'ConfigurationManager.SimulationData') -> None: """ Parses the configuration data (*sim_conf*) associated with a simulation (*sim_name*). Instantiate the components associated with each simulation. @@ -446,7 +491,11 @@ def _initialize_sim(self, sim_data: "ConfigurationManager.SimulationData") -> No continue comp_conf = sim_conf[comp_ref] except Exception: - self.fwk.exception('Error accessing configuration section for ' + 'component %s in simulation %s', comp_ref, sim_name) + self.fwk.exception( + 'Error accessing configuration section for ' + 'component %s in simulation %s', + comp_ref, + sim_name, + ) sys.exit(1) conf_fields = set(comp_conf.keys()) @@ -469,9 +518,7 @@ def _initialize_sim(self, sim_data: "ConfigurationManager.SimulationData") -> No else: comp_conf['BIN_PATH'] = comp_conf['BIN_DIR'] if not self.required_fields.issubset(conf_fields): - msg = 'Error: missing required entries {} in simulation {} component {} configuration section'.format( - list(self.required_fields - conf_fields), sim_name, comp_ref - ) + msg = f'Error: missing required entries {list(self.required_fields - conf_fields)} in simulation {sim_name} component {comp_ref} configuration section' self.fwk.critical(msg) raise RuntimeError(msg) component_id = self._create_component(comp_conf, sim_data) @@ -482,13 +529,18 @@ def _initialize_sim(self, sim_data: "ConfigurationManager.SimulationData") -> No sim_data.init_comp = component_id if sim_data.driver_comp is None: - msg = 'Missing DRIVER specification in config file for simulation {}'.format(sim_data.sim_name) + msg = f'Missing DRIVER specification in config file for simulation {sim_data.sim_name}' self.fwk.critical(msg) raise RuntimeError(msg) if sim_data.init_comp is None: - self.fwk.warning('Missing INIT specification in ' + 'config file for simulation %s', sim_data.sim_name) + self.fwk.warning( + 'Missing INIT specification in ' + 'config file for simulation %s', + sim_data.sim_name, + ) - def _create_component(self, comp_conf: dict[str, Any], sim_data: "ConfigurationManager.SimulationData") -> ComponentID: + def _create_component( + self, comp_conf: dict[str, Any], sim_data: 'ConfigurationManager.SimulationData' + ) -> ComponentID: """ Create component and populate it with the information from the component's configuration section. @@ -505,8 +557,14 @@ def _create_component(self, comp_conf: dict[str, Any], sim_data: "ConfigurationM spec.loader.exec_module(module) component_class = getattr(module, class_name) except (FileNotFoundError, AttributeError): - self.fwk.error('Error in configuration file : NAME = %s SCRIPT = %s', comp_conf['NAME'], comp_conf['SCRIPT']) - self.fwk.exception('Error instantiating IPS component %s From %s', class_name, script) + self.fwk.error( + 'Error in configuration file : NAME = %s SCRIPT = %s', + comp_conf['NAME'], + comp_conf['SCRIPT'], + ) + self.fwk.exception( + 'Error instantiating IPS component %s From %s', class_name, script + ) raise else: try: @@ -523,11 +581,15 @@ def _create_component(self, comp_conf: dict[str, Any], sim_data: "ConfigurationM fwk_inq = self.fwk.get_inq() log_pipe_name = sim_data.log_pipe_name - services_proxy = ServicesProxy(self.fwk, fwk_inq, svc_response_q, sim_data.sim_conf, log_pipe_name) + services_proxy = ServicesProxy( + self.fwk, fwk_inq, svc_response_q, sim_data.sim_conf, log_pipe_name + ) new_component = component_class(services_proxy, comp_conf) new_component.__initialize__(component_id, invocation_q, sim_data.start_time) services_proxy.__initialize__(new_component) - self.comp_registry.addEntry(component_id, svc_response_q, invocation_q, new_component, services_proxy, comp_conf) + self.comp_registry.add_entry( + component_id, svc_response_q, invocation_q, new_component, services_proxy, comp_conf + ) p = Process(target=new_component.__run__) p.start() sim_data.process_list.append(p) @@ -537,7 +599,7 @@ def _create_component(self, comp_conf: dict[str, Any], sim_data: "ConfigurationM def get_component_map(self) -> dict[str, list[ComponentID]]: """ Return a dictionary of simulation names and lists of component - references. (May only be the driver, and init (if present)???) + references. (May only be the Driver, and init (if present)???) """ sim_comps = {} for sim_name in self.sim_map: @@ -583,7 +645,9 @@ def get_sim_parameter(self, sim_name: str, param: str) -> Any: val = sim_data.sim_conf[param] except KeyError: val = self.platform_conf[param] - self.fwk.debug('Returning value = %s for config parameter %s in simulation %s', val, param, sim_name) + self.fwk.debug( + 'Returning value = %s for config parameter %s in simulation %s', val, param, sim_name + ) return val def get_sim_names(self) -> list[str]: @@ -600,20 +664,22 @@ def process_service_request(self, msg: Any) -> Any: self.fwk.debug('Configuration Manager received message: %s', str(msg.__dict__)) sim_name = msg.sender_id.get_sim_name() method = getattr(self, msg.target_method) - self.fwk.debug('Configuration manager dispatching method %s on simulation %s', method, sim_name) + self.fwk.debug( + 'Configuration manager dispatching method %s on simulation %s', method, sim_name + ) retval = method(sim_name, *msg.args) return retval def create_simulation( self, sim_name: str, - config_file: Union[str, os.PathLike[str]], - override: Optional[dict[str, Any]], + config_file: str | os.PathLike[str], + override: dict[str, Any] | None, sub_workflow: bool = False, - ) -> tuple[str, Optional[ComponentID], Optional[ComponentID]]: + ) -> tuple[str, ComponentID | None, ComponentID | None]: try: conf = ConfigObj(config_file, interpolation='template', file_error=True) - except IOError: + except OSError: self.fwk.exception('Error opening config file %s: ', config_file) raise except SyntaxError: @@ -664,7 +730,9 @@ def create_simulation( self.sim_name_list.append(sim_name) self.sim_root_list.append(sim_root) self.log_file_list.append(log_file) - new_sim = self.SimulationData(sim_name, start_time=self.fwk.start_time if sub_workflow else time.time()) + new_sim = self.SimulationData( + sim_name, start_time=self.fwk.start_time if sub_workflow else time.time() + ) new_sim.sim_conf = conf new_sim.config_file = config_file new_sim.sim_root = sim_root @@ -672,7 +740,9 @@ def create_simulation( if not sub_workflow: new_sim.portal_sim_name = sim_name new_sim.log_pipe_name = f'{tempfile.gettempdir()}/ips_{uuid.uuid4()}.logpipe' - self.log_dynamic_sim_queue.put('CREATE_SIM %s %s' % (new_sim.log_pipe_name, new_sim.log_file)) + self.log_dynamic_sim_queue.put( + 'CREATE_SIM %s %s' % (new_sim.log_pipe_name, new_sim.log_file) + ) else: new_sim.portal_sim_name = parent_sim.portal_sim_name new_sim.log_pipe_name = parent_sim.log_pipe_name @@ -702,7 +772,9 @@ def get_config_parameter(self, sim_name: str, param: str) -> Any: """ return self.get_sim_parameter(sim_name, param) - def set_config_parameter(self, sim_name: str, param: str, value: Any, target_sim_name: str) -> Any: + def set_config_parameter( + self, sim_name: str, param: str, value: Any, target_sim_name: str + ) -> Any: """ Set the configuration parameter *param* to value *value* in *target_sim_name*. If *target_sim_name* is the framework, all @@ -753,7 +825,7 @@ def terminate_sim(self, sim_name: str) -> None: except Exception: pass for comp_id in all_sim_components: - self.comp_registry.removeEntry(comp_id) + self.comp_registry.remove_entry(comp_id) sim_data.logger = None sim_data.process_list = [] self.finished_sim_map[sim_name] = sim_data diff --git a/ipsframework/convert_log_function.py b/ipsframework/convert_log_function.py index aab745bd..b47f2d51 100644 --- a/ipsframework/convert_log_function.py +++ b/ipsframework/convert_log_function.py @@ -7,8 +7,28 @@ def parse_log_line(line: str): tokens = line.split() - ret_fields = ['event_time', 'event_num', 'eventtype', 'code', 'state', 'walltime', 'phystimestamp', 'comment'] - field_names = ['code', 'eventtype', 'ok', 'walltime', 'state', 'comment', 'sim_name', 'portal_runid', 'seqnum', 'phystimestamp'] + ret_fields = [ + 'event_time', + 'event_num', + 'eventtype', + 'code', + 'state', + 'walltime', + 'phystimestamp', + 'comment', + ] + field_names = [ + 'code', + 'eventtype', + 'ok', + 'walltime', + 'state', + 'comment', + 'sim_name', + 'portal_runid', + 'seqnum', + 'phystimestamp', + ] val_dict: dict[str, str] = {} @@ -16,7 +36,10 @@ def parse_log_line(line: str): val_dict['event_time'] = tokens[1] start = {s: line.find(s) + len(s + '=') for s in field_names} - end = {s: line.find("'", start[s] + 1) if line[start[s]] == "'" else line.find(' ', start[s] + 1) for s in field_names} + end = { + s: line.find("'", start[s] + 1) if line[start[s]] == "'" else line.find(' ', start[s] + 1) + for s in field_names + } for field_name, position in end.items(): if position == -1: end[field_name] = len(line) @@ -35,7 +58,16 @@ def convert_logdata_to_html(logdata: str): if 'IPS_RESOURCE_ALLOC' not in line and 'IPS_START' not in line and 'IPS_END' not in line: tmp = parse_log_line(line) tokens.append(tmp) - header = ['Time', 'Sequence Num', 'Type', 'Code', 'State', 'Wall Time', 'Physics Time', 'Comment'] + header = [ + 'Time', + 'Sequence Num', + 'Type', + 'Code', + 'State', + 'Wall Time', + 'Physics Time', + 'Comment', + ] html_page = HTML.table(tokens, header_row=header) return html_page diff --git a/ipsframework/dakota_bridge.py b/ipsframework/dakota_bridge.py index caea4cf8..414092b7 100644 --- a/ipsframework/dakota_bridge.py +++ b/ipsframework/dakota_bridge.py @@ -45,8 +45,10 @@ def step(self, timestamp=0, **keywords): # pragma: no cover """ # parse file try: - self.old_master_conf = ConfigObj(self.config_file, interpolation='template', file_error=True) - except (IOError, SyntaxError): + self.old_master_conf = ConfigObj( + self.config_file, interpolation='template', file_error=True + ) + except (OSError, SyntaxError): raise self.sim_root = services.get_config_param('SIM_ROOT') self.sim_name = services.get_config_param('SIM_NAME') @@ -55,11 +57,17 @@ def step(self, timestamp=0, **keywords): # pragma: no cover sim_config_files = [] idx = 0 - print('%s About to Create Listener %s' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), str(self.socket_address))) + print( + '%s About to Create Listener %s' + % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), str(self.socket_address)) + ) sys.stdout.flush() listener = Listener(str(self.socket_address), 'AF_UNIX') self.services.warning('Created listener %s', str(self.socket_address)) - print('%s Created Listener %s' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), str(self.socket_address))) + print( + '%s Created Listener %s' + % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), str(self.socket_address)) + ) sys.stdout.flush() sim_cache = {} sock_fileno = listener._listener._socket.fileno() @@ -103,7 +111,12 @@ def step(self, timestamp=0, **keywords): # pragma: no cover try: msg = conn.recv() except Exception as inst: - print('%s EXCEPTION in conn.recv(): failed connections = ' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime())), type(inst), str(inst)) + print( + '%s EXCEPTION in conn.recv(): failed connections = ' + % (time.strftime('%b %d %Y %H:%M:%S', time.localtime())), + type(inst), + str(inst), + ) if failed_connections > 5: raise else: @@ -129,7 +142,9 @@ def step(self, timestamp=0, **keywords): # pragma: no cover file_name = os.path.join(self.sim_root, 'simulation_%s.conf' % (instance_id)) self.old_master_conf.filename = file_name - self.old_master_conf['SIM_ROOT'] = os.path.join(self.sim_root, 'simulation_%s' % (instance_id)) + self.old_master_conf['SIM_ROOT'] = os.path.join( + self.sim_root, 'simulation_%s' % (instance_id) + ) self.old_master_conf['SIM_NAME'] = self.sim_name + '_%s' % (instance_id) self.old_master_conf['LOG_FILE'] = self.sim_logfile + '_%s' % (instance_id) self.old_master_conf['OUT_REDIRECT'] = 'TRUE' @@ -141,10 +156,15 @@ def step(self, timestamp=0, **keywords): # pragma: no cover try: os.makedirs(self.old_master_conf['SIM_ROOT'], exist_ok=True) except OSError as oserr: - print('Error creating Simulation directory %s : %d %s' % (self.old_master_conf['SIM_ROOT'], oserr.errno, oserr.strerror)) + print( + 'Error creating Simulation directory %s : %d %s' + % (self.old_master_conf['SIM_ROOT'], oserr.errno, oserr.strerror) + ) raise if first_sim: - summary_file = open(os.path.join(self.sim_root, 'SIMULATION_LIST.%s' % (dakota_runid)), 'a', 1) + summary_file = open( + os.path.join(self.sim_root, 'SIMULATION_LIST.%s' % (dakota_runid)), 'a', 1 + ) param_file = os.path.join(self.old_master_conf['SIM_ROOT'], 'parameters.conf') param_string = '' @@ -179,8 +199,10 @@ def finalize(self, timestamp=0, **keywords): # Driver finalize - nothing to be done pass - def process_event(self, topicName, theEvent): - event_body = theEvent.getBody() + def process_event(self, topic_name, the_event): + event_body = the_event.get_body() self.events_received.append(event_body) self.services.debug('In Component: Just received %s', str(event_body)) - self.services.debug('In Component: There are %d events in self.events_received', len(self.events_received)) + self.services.debug( + 'In Component: There are %d events in self.events_received', len(self.events_received) + ) diff --git a/ipsframework/dataManager.py b/ipsframework/data_manager.py similarity index 85% rename from ipsframework/dataManager.py rename to ipsframework/data_manager.py index a7df4c8d..91b019c3 100644 --- a/ipsframework/dataManager.py +++ b/ipsframework/data_manager.py @@ -57,7 +57,7 @@ def stage_state(self, msg): source_dir = msg.args[1] target_dir = msg.args[2] try: - ipsutil.copyFiles(source_dir, state_files, target_dir) + ipsutil.copy_files(source_dir, state_files, target_dir) except Exception: self.fwk.exception('Error staging plasma state files to directory %s', target_dir) raise @@ -78,7 +78,7 @@ def update_state(self, msg): source_dir = msg.args[1] target_dir = msg.args[2] try: - ipsutil.copyFiles(source_dir, state_files, target_dir) + ipsutil.copy_files(source_dir, state_files, target_dir) except Exception: self.fwk.exception('Error updating state files from directory %s', source_dir) raise @@ -114,7 +114,11 @@ def merge_current_plasma_state(self, msg): self.fwk.exception('Error opening log file %s : using stdout', log_fullpath) try: - retval = subprocess.call([update_state, '-input', target_state_file, '-updates', partial_state_file], stdout=merge_stdout, stderr=subprocess.STDOUT) + retval = subprocess.call( + [update_state, '-input', target_state_file, '-updates', partial_state_file], + stdout=merge_stdout, + stderr=subprocess.STDOUT, + ) except Exception: self.fwk.exception('Error calling update_state - probably not found in $PATH') raise @@ -122,8 +126,11 @@ def merge_current_plasma_state(self, msg): if retval != 0: return retval try: - ipsutil.copyFiles(plasma_work_dir, current_plasma_state, component_work_dir) + ipsutil.copy_files(plasma_work_dir, current_plasma_state, component_work_dir) except Exception: - self.fwk.exception('Error refreshing local copy of current plasma state file in directory %s', component_work_dir) + self.fwk.exception( + 'Error refreshing local copy of current plasma state file in directory %s', + component_work_dir, + ) raise return 0 diff --git a/ipsframework/eventService.py b/ipsframework/eventService.py deleted file mode 100644 index 1d693304..00000000 --- a/ipsframework/eventService.py +++ /dev/null @@ -1,326 +0,0 @@ -# ------------------------------------------------------------------------------- -# Copyright 2006-2022 UT-Battelle, LLC. See LICENSE for more information. -# ------------------------------------------------------------------------------- -""" -This file hosts the central event service and is not directly accessible to IPS. -The cca_es_spec.py file provides a CCA-style event service interface to IPS, -with calls on the interface being routed here via the proxy. The CCA event -interface is straightforwardly mapped onto matching methods in this file. -""" - -from .cca_es_spec import Event, EventServiceException, Topic -from .debug import debug -from .topicManager import TopicManager - - -class EventService: - def __init__(self, fwk=None): - """ - The following two data structures are at the heart of the event service. - The design and implementation of the event service becomes clear from - the composition of these two structures. - - - topicDirectory is a map, where topic is identified by - topicName and events are stored in an TopicManager object. The TopicManager - class appears in topicManager.py and holds events posted to a topic. - It also maintains the list of listeners subscribed to that topic. - The topicDirectory is a flat listing of topics. A topic hierarchy can be - built as an adjunct structure, without sacrificing topicDirectory. The - topicDirectory facilitates very easy posting and propagation of events to - topics and listeners respectively. The actual mechanics of it is hidden - inside the TopicManager class. - - - subscriberDirectory is a three-level nested map: - subscriberid - subscriptionName - listenerKey - listenerid. - - This structure reflects the fact that a subscriber can create multiple - subscriptions, a subscription in turn can comprise of multiple listenerKeys - , and a listenerKey is linked with one listenerid or listener object. - """ - - """ Singleton pattern """ - - self.topicDirectory = {} - self.subscriberDirectory = {} - self.numSubscribers = 0 - self.numListeners = 0 - self.fwk = fwk - if fwk: - service_methods = [ - 'getTopic', - 'existsTopic', - 'registerSubscriber', - 'unregisterSubscriber', - 'getSubscription', - 'processEvents', - 'sendEvent', - 'createListener', - 'registerEventListener', - 'unregisterEventListener', - 'removeSubscription', - ] - fwk.register_service_handler(service_methods, self.process_service_request) - - def _print_stats(self): - if self.fwk: - self.fwk.debug(':::::::::TOPIC-WISE EVENT STATS:::::::::') - for topicName, topic in self.topicDirectory.items(): - self.fwk.debug('%s = %s', topicName, topic.getEventStats()) - self.fwk.debug('::::::::::::::::::::::::::::::::::::::::') - - def process_service_request(self, msg): - method = getattr(self, msg.target_method) - return method(*msg.args) - - """""" """PublisherEventService methods start here""" """""" - - def getTopic(self, topicName): - """Add an entry to the topicDirectory for a new topic.""" - if topicName not in self.topicDirectory: - debug.output('getTopic %s' % topicName) - self.topicDirectory[topicName] = TopicManager() - return Topic(topicName) - - def existsTopic(self, topicName): - return topicName in self.topicDirectory - - """""" """PublisherEventService methods end here""" """""" - - """""" """SubscriberEventService methods start here""" """""" - - def registerSubscriber(self): - self.numSubscribers += 1 - subscriberid = self.numSubscribers - self.subscriberDirectory[subscriberid] = {} - debug.output('Subscriber registered', subscriberid) - return subscriberid - - """ - unregisterSubscriber is called when a subscriber object is being deleted. - This involves removal of the subscriber's listener entries from - listenerDirectory as well as all the TopicManagers corresponding to the - topics on which the subscriber is registered. Finally the subscriber record - in subscriberDirectory is purged. - """ - - def unregisterSubscriber(self, subscriberid): - listenerList = [] - if subscriberid in self.subscriberDirectory: - debug.output('\n\n------Subscriber is unregistering', subscriberid) - - """ - Step through all the listeners for the subscriber in turn, - first unregistering a listener from all subscribed topics and then - deleting it from the listenerDirectory. - """ - for subscriptionName in self.subscriberDirectory[subscriberid]: - for listenerKey in self.subscriberDirectory[subscriberid][subscriptionName]: - listenerid = self.subscriberDirectory[subscriberid][subscriptionName][listenerKey] - debug.output('Unregistering listener on listenerKey %s, subscription %s' % (listenerKey, subscriptionName), listenerid, subscriberid) - topicList = self._mapListenerKeytoTopicList(subscriptionName, listenerKey) - for topicName in topicList: - self.topicDirectory[topicName].unregisterListener(listenerid) - debug.output('Listener on listenerKey %s, subscription %s unregistered' % (listenerKey, subscriptionName), listenerid, subscriberid) - listenerList.append(listenerid) - """ Remove the subscriber entry in subscriberDirectory. """ - del self.subscriberDirectory[subscriberid] - debug.output('Subscriber unregistered', subscriberid) - else: - raise EventServiceException('Subscriber not recognized.') - return listenerList - - def getSubscription(self, subscriberid, subscriptionName): - if subscriberid in self.subscriberDirectory: - """ - We do not allow for the possibility that a subscriptionName may mean more - than one topic name. May need to be changed in future for greater - flexibility. Note that we do a getTopic here as a subscribe could happen - before any publisher creates the particular topic. - """ - self.getTopic(subscriptionName) - - if subscriptionName not in self.subscriberDirectory[subscriberid]: - self.subscriberDirectory[subscriberid][subscriptionName] = {} - debug.output('Subscriber subscribed to %s' % subscriptionName, subscriberid) - - """ - A Subscription object cannot be safely returned without screwing - up automatic object tracking for cleaning up out-of-scope - subscriptions on the component side. The reason being a component - is handed a copy of the object returned from here, while the - object itself goes out-of-scope immediately, thereby triggering - a cleanup of the associated subscription, with the undesirable - result of a subscription becoming invalid even while the - component-side Subscription object is still in use. - """ - # return Subscription(subscriberid,subscriptionName) - else: - """ Should we permit duplicate subscription requests? """ - raise EventServiceException('Duplicate subscription request.') - else: - raise EventServiceException('Subscriber not recognized.') - - """ - A subscriber performs a processEvents to learn about events posted to its - topics of interest. This requires traversing the complete subscriber record - in subscriberDirectory, and doing a processEvent for every event sent since - the last such call to topics on which the subscriber is registered. - """ - - def processEvents(self, subscriberid): - eventList = {} - if subscriberid in self.subscriberDirectory: - for subscriptionName in self.subscriberDirectory[subscriberid]: - for listenerKey in self.subscriberDirectory[subscriberid][subscriptionName]: - listenerid = self.subscriberDirectory[subscriberid][subscriptionName][listenerKey] - """ This check is required to allow the _same_ listener to handle different topics. """ - if listenerid not in eventList: - eventList[listenerid] = {} - topicList = self._mapListenerKeytoTopicList(subscriptionName, listenerKey) - for topicName in topicList: - eventList[listenerid][topicName] = self.topicDirectory[topicName].getEventListForListener(listenerid) - else: - raise EventServiceException('Subscriber not recognized.') - return eventList - - """""" """SubscriberEventService methods end here""" """""" - - """""" """Topic methods start here""" """""" - - """ - sendEvent adds an event to the topic's TopicManager object. - """ - - def sendEvent(self, topicName, eventName, eventBody): - if topicName in self.topicDirectory: - eventHeader = {} - eventHeader[eventName] = eventName - theEvent = Event(eventHeader, eventBody) - debug.output('Event %s sent to topic %s' % (theEvent, topicName)) - self.topicDirectory[topicName].sendEvent(theEvent) - else: - raise EventServiceException('Topic not recognized.') - - """""" """Topic methods end here""" """""" - - """""" """EventListener methods start here""" """""" - - def createListener(self): - self.numListeners += 1 - listenerid = self.numListeners - debug.output('Listener created', listenerid) - return listenerid - - """""" """EventListener methods end here""" """""" - - """""" """Subscription methods start here""" """""" - - """ - registerEventListener adds a listener to its subscriber's subscriberDirectory - record, the TopicManager for the associated topic, and the listenerDirectory. - """ - - def registerEventListener(self, subscriberid, subscriptionName, listenerKey, listenerid): - if subscriberid in self.subscriberDirectory: - if subscriptionName in self.subscriberDirectory[subscriberid]: - """ - We do not allow for the possibility that a subscriptionName may mean - more than one topic name. May need to be changed in future for greater - flexibility. - """ - if subscriptionName == listenerKey: - if listenerKey not in self.subscriberDirectory[subscriberid][subscriptionName]: - debug.output('Registering listener on listenerKey %s, subscription %s' % (listenerKey, subscriptionName), listenerid, subscriberid) - topicList = self._mapListenerKeytoTopicList(subscriptionName, listenerKey) - for topicName in topicList: - self.topicDirectory[topicName].registerListener(listenerid) - self.subscriberDirectory[subscriberid][subscriptionName][listenerKey] = listenerid - else: - """ - Should we allow a listenerKey to be re-registered before first - unregistering? - """ - raise EventServiceException('Duplicate event listener.') - else: - raise EventServiceException('Listener key not recognized.') - else: - raise EventServiceException('Subscription not recognized.') - else: - raise EventServiceException('Subscriber not recognized.') - - """ - unregisterEventListener removes a listener from its subscriber's - subscriberDirectory record, the TopicManager for the associated topic, and - the listenerDirectory. - """ - - def unregisterEventListener(self, subscriberid, subscriptionName, listenerKey): - listenerid = -1 - if subscriberid in self.subscriberDirectory: - if subscriptionName in self.subscriberDirectory[subscriberid]: - if listenerKey in self.subscriberDirectory[subscriberid][subscriptionName]: - listenerid = self.subscriberDirectory[subscriberid][subscriptionName][listenerKey] - debug.output('Unregistering listener on listenerKey %s, subscription %s' % (listenerKey, subscriptionName), listenerid, subscriberid) - topicList = self._mapListenerKeytoTopicList(subscriptionName, listenerKey) - for topicName in topicList: - self.topicDirectory[topicName].unregisterListener(listenerid) - del self.subscriberDirectory[subscriberid][subscriptionName][listenerKey] - debug.output('Listener on listenerKey %s, subscription %s unregistered' % (listenerKey, subscriptionName), listenerid, subscriberid) - else: - raise EventServiceException('Listener key not recognized.') - """ - Do not raise exception if subscriberid/subscriptionName turn out to be - invalid as this can very well happen if subscriber/subscription object - is garbage collected before its listener object. In such a scenario, - unregisterSubscriber/removeSubscription will clean up this listener also. - """ - return listenerid - - """ - removeSubscription is called when a subscription object is being deleted. - It unregisters associated listeners from their respective TopicManagers and the - listenerDirectory, and then removes its subscription information from the - subscriber entry in subscriberDirectory. - """ - - def removeSubscription(self, subscriberid, subscriptionName): - listenerList = [] - if subscriberid in self.subscriberDirectory: - if subscriptionName in self.subscriberDirectory[subscriberid]: - debug.output("\n\n------Subscriber's subscription to %s is being removed" % subscriptionName, subscriberid) - for listenerKey in self.subscriberDirectory[subscriberid][subscriptionName]: - listenerid = self.subscriberDirectory[subscriberid][subscriptionName][listenerKey] - debug.output('Unregistering listener on listenerKey %s, subscription %s' % (listenerKey, subscriptionName), listenerid, subscriberid) - topicList = self._mapListenerKeytoTopicList(subscriptionName, listenerKey) - for topicName in topicList: - self.topicDirectory[topicName].unregisterListener(listenerid) - debug.output('Listener on listenerKey %s, subscription %s unregistered' % (listenerKey, subscriptionName), listenerid, subscriberid) - listenerList.append(listenerid) - del self.subscriberDirectory[subscriberid][subscriptionName] - debug.output("Subscriber's subscription to %s removed" % subscriptionName, subscriberid) - """ - Do not raise exception if subscriberid/subscriptionName turn out to be - invalid as this can very well happen if subscriber object is garbage - collected before its subscription object. In such a scenario, - unregisterSubscriber will clean up this subscription as well. - """ - return listenerList - - """""" """Subscription methods end here""" """""" - - """""" """Methods internal to the event service start here""" """""" - - """ - A listenerKey may specify a bunch of topics using wildcarding. - Currently this is not supported. Need a more rigorous design to allow - wildcarding. - """ - - def _mapListenerKeytoTopicList(self, subscriptionName, listenerKey): - topicList = [] - topicList.append(listenerKey) - return topicList - - """""" """Methods internal to the event service end here""" """""" diff --git a/ipsframework/eventServiceProxy.py b/ipsframework/eventServiceProxy.py deleted file mode 100644 index 84abba53..00000000 --- a/ipsframework/eventServiceProxy.py +++ /dev/null @@ -1,167 +0,0 @@ -# ------------------------------------------------------------------------------- -# Copyright 2006-2022 UT-Battelle, LLC. See LICENSE for more information. -# ------------------------------------------------------------------------------- - - -class EventServiceProxy: - def getTopic(self, topicName): - pass - - def existsTopic(self, topicName): - pass - - def registerSubscriber(self): - pass - - def unregisterSubscriber(self, subscriberid): - pass - - def getSubscription(self, subscriberid, subscriptionName): - pass - - def processEvents(self, subscriberid): - pass - - def sendEvent(self, topicName, eventName, eventBody): - pass - - def createListener(self): - pass - - def registerEventListener(self, subscriberid, subscriptionName, listenerKey, listenerid, refListener): - pass - - def unregisterEventListener(self, subscriberid, subscriptionName, listenerKey): - pass - - def removeSubscription(self, subscriberid, subscriptionName): - pass - - -# TODO: Is eventService.py the right placeholder for this class?# -class EventServiceFwkProxy(EventServiceProxy): - def __init__(self, event_service): - self.event_service = event_service - self.listenerDirectory = {} - - def getTopic(self, topicName): - return self.event_service.getTopic(topicName) - - def existsTopic(self, topicName): - return self.event_service.existsTopic(topicName) - - def registerSubscriber(self): - return self.event_service.registerSubscriber() - - def unregisterSubscriber(self, subscriberid): - listenerList = self.event_service.unregisterSubscriber(subscriberid) - for listenerid in listenerList: - self._removeEventListener(listenerid) - - def getSubscription(self, subscriberid, subscriptionName): - self.event_service.getSubscription(subscriberid, subscriptionName) - - def processEvents(self, subscriberid): - eventList = self.event_service.processEvents(subscriberid) - for listenerid in eventList: - for topicName in eventList[listenerid]: - for theEvent in eventList[listenerid][topicName]: - self.listenerDirectory[listenerid].processEvent(topicName, theEvent) - - def sendEvent(self, topicName, eventName, eventBody): - self.event_service.sendEvent(topicName, eventName, eventBody) - - def createListener(self): - return self.event_service.createListener() - - def registerEventListener(self, subscriberid, subscriptionName, listenerKey, listenerid, refListener): - self.event_service.registerEventListener(subscriberid, subscriptionName, listenerKey, listenerid) - self._addEventListener(listenerid, refListener) - - def unregisterEventListener(self, subscriberid, subscriptionName, listenerKey): - listenerid = self.event_service.unregisterEventListener(subscriberid, subscriptionName, listenerKey) - self._removeEventListener(listenerid) - - def removeSubscription(self, subscriberid, subscriptionName): - listenerList = self.event_service.removeSubscription(subscriberid, subscriptionName) - for listenerid in listenerList: - self._removeEventListener(listenerid) - - def _addEventListener(self, listenerid, refListener): - if listenerid not in self.listenerDirectory: - self.listenerDirectory[listenerid] = refListener - - def _removeEventListener(self, listenerid): - if listenerid in self.listenerDirectory: - del self.listenerDirectory[listenerid] - - -# TODO: Is services.py the right placeholder for this class?# -class EventServiceCmpProxy(EventServiceProxy): - def __init__(self, service_proxy): - self.service_proxy = service_proxy - self.listenerDirectory = {} - - def getTopic(self, topicName): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'getTopic', topicName) - return self.service_proxy._get_service_response(msg_id, True) - - def existsTopic(self, topicName): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'existsTopic', topicName) - return self.service_proxy._get_service_response(msg_id, True) - - def registerSubscriber(self): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'registerSubscriber') - return self.service_proxy._get_service_response(msg_id, True) - - def unregisterSubscriber(self, subscriberid): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'unregisterSubscriber', subscriberid) - listenerList = self.service_proxy._get_service_response(msg_id, True) - for listenerid in listenerList: - self._removeEventListener(listenerid) - - def getSubscription(self, subscriberid, subscriptionName): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'getSubscription', subscriberid, subscriptionName) - self.service_proxy._get_service_response(msg_id, True) - - def processEvents(self, subscriberid): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'processEvents', subscriberid) - eventList = self.service_proxy._get_service_response(msg_id, True) - for listenerid in eventList: - for topicName in eventList[listenerid]: - for theEvent in eventList[listenerid][topicName]: - self.listenerDirectory[listenerid].processEvent(topicName, theEvent) - - def sendEvent(self, topicName, eventName, eventBody): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'sendEvent', topicName, eventName, eventBody) - self.service_proxy._get_service_response(msg_id, True) - - def createListener(self): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'createListener') - return self.service_proxy._get_service_response(msg_id, True) - - def registerEventListener(self, subscriberid, subscriptionName, listenerKey, listenerid, refListener): - msg_id = self.service_proxy._invoke_service( - self.service_proxy.fwk.component_id, 'registerEventListener', subscriberid, subscriptionName, listenerKey, listenerid - ) - self.service_proxy._get_service_response(msg_id, True) - self._addEventListener(listenerid, refListener) - - def unregisterEventListener(self, subscriberid, subscriptionName, listenerKey): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'unregisterEventListener', subscriberid, subscriptionName, listenerKey) - listenerid = self.service_proxy._get_service_response(msg_id, True) - self._removeEventListener(listenerid) - - def removeSubscription(self, subscriberid, subscriptionName): - msg_id = self.service_proxy._invoke_service(self.service_proxy.fwk.component_id, 'removeSubscription', subscriberid, subscriptionName) - listenerList = self.service_proxy._get_service_response(msg_id, True) - for listenerid in listenerList: - self._removeEventListener(listenerid) - - def _addEventListener(self, listenerid, refListener): - if listenerid not in self.listenerDirectory: - self.listenerDirectory[listenerid] = refListener - - def _removeEventListener(self, listenerid): - if listenerid in self.listenerDirectory: - del self.listenerDirectory[listenerid] diff --git a/ipsframework/event_service.py b/ipsframework/event_service.py new file mode 100644 index 00000000..dd3c6170 --- /dev/null +++ b/ipsframework/event_service.py @@ -0,0 +1,392 @@ +# ------------------------------------------------------------------------------- +# Copyright 2006-2022 UT-Battelle, LLC. See LICENSE for more information. +# ------------------------------------------------------------------------------- +""" +This file hosts the central event service and is not directly accessible to IPS. +The cca_es_spec.py file provides a CCA-style event service interface to IPS, +with calls on the interface being routed here via the proxy. The CCA event +interface is straightforwardly mapped onto matching methods in this file. +""" + +from .cca_es_spec import Event, EventServiceError, Topic +from .debug import debug +from .topic_manager import TopicManager + + +class EventService: + def __init__(self, fwk=None): + """ + The following two data structures are at the heart of the event service. + The design and implementation of the event service becomes clear from + the composition of these two structures. + + + topicDirectory is a map, where topic is identified by + topic_name and events are stored in an TopicManager object. The TopicManager + class appears in topic_manager.py and holds events posted to a topic. + It also maintains the list of listeners subscribed to that topic. + The topicDirectory is a flat listing of topics. A topic hierarchy can be + built as an adjunct structure, without sacrificing topicDirectory. The + topicDirectory facilitates very easy posting and propagation of events to + topics and listeners respectively. The actual mechanics of it is hidden + inside the TopicManager class. + + + subscriberDirectory is a three-level nested map: + subscriberid - subscription_name - listener_key - listenerid. + + This structure reflects the fact that a subscriber can create multiple + subscriptions, a subscription in turn can comprise of multiple listenerKeys + , and a listener_key is linked with one listenerid or listener object. + """ + + """ Singleton pattern """ + + self.topicDirectory = {} + self.subscriberDirectory = {} + self.numSubscribers = 0 + self.numListeners = 0 + self.fwk = fwk + if fwk: + service_methods = [ + 'get_topic', + 'exists_topic', + 'register_subscriber', + 'unregister_subscriber', + 'get_subscription', + 'process_events', + 'send_event', + 'create_listener', + 'register_event_listener', + 'unregister_event_listener', + 'remove_subscription', + ] + fwk.register_service_handler(service_methods, self.process_service_request) + + def _print_stats(self): + if self.fwk: + self.fwk.debug(':::::::::TOPIC-WISE EVENT STATS:::::::::') + for topic_name, topic in self.topicDirectory.items(): + self.fwk.debug('%s = %s', topic_name, topic.get_event_stats()) + self.fwk.debug('::::::::::::::::::::::::::::::::::::::::') + + def process_service_request(self, msg): + method = getattr(self, msg.target_method) + return method(*msg.args) + + """""" """PublisherEventService methods start here""" """""" + + def get_topic(self, topic_name): + """Add an entry to the topicDirectory for a new topic.""" + if topic_name not in self.topicDirectory: + debug.output('get_topic %s' % topic_name) + self.topicDirectory[topic_name] = TopicManager() + return Topic(topic_name) + + def exists_topic(self, topic_name): + return topic_name in self.topicDirectory + + """""" """PublisherEventService methods end here""" """""" + + """""" """SubscriberEventService methods start here""" """""" + + def register_subscriber(self): + self.numSubscribers += 1 + subscriberid = self.numSubscribers + self.subscriberDirectory[subscriberid] = {} + debug.output('Subscriber registered', subscriberid) + return subscriberid + + """ + unregister_subscriber is called when a subscriber object is being deleted. + This involves removal of the subscriber's listener entries from + listenerDirectory as well as all the TopicManagers corresponding to the + topics on which the subscriber is registered. Finally the subscriber record + in subscriberDirectory is purged. + """ + + def unregister_subscriber(self, subscriberid): + listener_list = [] + if subscriberid in self.subscriberDirectory: + debug.output('\n\n------Subscriber is unregistering', subscriberid) + + """ + Step through all the listeners for the subscriber in turn, + first unregistering a listener from all subscribed topics and then + deleting it from the listenerDirectory. + """ + for subscription_name in self.subscriberDirectory[subscriberid]: + for listener_key in self.subscriberDirectory[subscriberid][subscription_name]: + listenerid = self.subscriberDirectory[subscriberid][subscription_name][ + listener_key + ] + debug.output( + 'Unregistering listener on listener_key %s, subscription %s' + % (listener_key, subscription_name), + listenerid, + subscriberid, + ) + topic_list = self._map_listener_key_to_topic_list( + subscription_name, listener_key + ) + for topic_name in topic_list: + self.topicDirectory[topic_name].unregister_listener(listenerid) + debug.output( + 'Listener on listener_key %s, subscription %s unregistered' + % (listener_key, subscription_name), + listenerid, + subscriberid, + ) + listener_list.append(listenerid) + """ Remove the subscriber entry in subscriberDirectory. """ + del self.subscriberDirectory[subscriberid] + debug.output('Subscriber unregistered', subscriberid) + else: + raise EventServiceError('Subscriber not recognized.') + return listener_list + + def get_subscription(self, subscriberid, subscription_name): + if subscriberid in self.subscriberDirectory: + """ + We do not allow for the possibility that a subscription_name may mean more + than one topic name. May need to be changed in future for greater + flexibility. Note that we do a get_topic here as a subscribe could happen + before any publisher creates the particular topic. + """ + self.get_topic(subscription_name) + + if subscription_name not in self.subscriberDirectory[subscriberid]: + self.subscriberDirectory[subscriberid][subscription_name] = {} + debug.output('Subscriber subscribed to %s' % subscription_name, subscriberid) + + """ + A Subscription object cannot be safely returned without screwing + up automatic object tracking for cleaning up out-of-scope + subscriptions on the component side. The reason being a component + is handed a copy of the object returned from here, while the + object itself goes out-of-scope immediately, thereby triggering + a cleanup of the associated subscription, with the undesirable + result of a subscription becoming invalid even while the + component-side Subscription object is still in use. + """ + # return Subscription(subscriberid,subscription_name) + else: + """ Should we permit duplicate subscription requests? """ + raise EventServiceError('Duplicate subscription request.') + else: + raise EventServiceError('Subscriber not recognized.') + + """ + A subscriber performs a process_events to learn about events posted to its + topics of interest. This requires traversing the complete subscriber record + in subscriberDirectory, and doing a process_event for every event sent since + the last such call to topics on which the subscriber is registered. + """ + + def process_events(self, subscriberid): + event_list = {} + if subscriberid in self.subscriberDirectory: + for subscription_name in self.subscriberDirectory[subscriberid]: + for listener_key in self.subscriberDirectory[subscriberid][subscription_name]: + listenerid = self.subscriberDirectory[subscriberid][subscription_name][ + listener_key + ] + """ This check is required to allow the _same_ listener to handle different topics. """ + if listenerid not in event_list: + event_list[listenerid] = {} + topic_list = self._map_listener_key_to_topic_list( + subscription_name, listener_key + ) + for topic_name in topic_list: + event_list[listenerid][topic_name] = self.topicDirectory[ + topic_name + ].get_event_list_for_listener(listenerid) + else: + raise EventServiceError('Subscriber not recognized.') + return event_list + + """""" """SubscriberEventService methods end here""" """""" + + """""" """Topic methods start here""" """""" + + """ + send_event adds an event to the topic's TopicManager object. + """ + + def send_event(self, topic_name, event_name, event_body): + if topic_name in self.topicDirectory: + event_header = {} + event_header[event_name] = event_name + the_event = Event(event_header, event_body) + debug.output('Event %s sent to topic %s' % (the_event, topic_name)) + self.topicDirectory[topic_name].send_event(the_event) + else: + raise EventServiceError('Topic not recognized.') + + """""" """Topic methods end here""" """""" + + """""" """EventListener methods start here""" """""" + + def create_listener(self): + self.numListeners += 1 + listenerid = self.numListeners + debug.output('Listener created', listenerid) + return listenerid + + """""" """EventListener methods end here""" """""" + + """""" """Subscription methods start here""" """""" + + """ + register_event_listener adds a listener to its subscriber's subscriberDirectory + record, the TopicManager for the associated topic, and the listenerDirectory. + """ + + def register_event_listener(self, subscriberid, subscription_name, listener_key, listenerid): + if subscriberid in self.subscriberDirectory: + if subscription_name in self.subscriberDirectory[subscriberid]: + """ + We do not allow for the possibility that a subscription_name may mean + more than one topic name. May need to be changed in future for greater + flexibility. + """ + if subscription_name == listener_key: + if ( + listener_key + not in self.subscriberDirectory[subscriberid][subscription_name] + ): + debug.output( + 'Registering listener on listener_key %s, subscription %s' + % (listener_key, subscription_name), + listenerid, + subscriberid, + ) + topic_list = self._map_listener_key_to_topic_list( + subscription_name, listener_key + ) + for topic_name in topic_list: + self.topicDirectory[topic_name].register_listener(listenerid) + self.subscriberDirectory[subscriberid][subscription_name][listener_key] = ( + listenerid + ) + else: + """ + Should we allow a listener_key to be re-registered before first + unregistering? + """ + raise EventServiceError('Duplicate event listener.') + else: + raise EventServiceError('Listener key not recognized.') + else: + raise EventServiceError('Subscription not recognized.') + else: + raise EventServiceError('Subscriber not recognized.') + + """ + unregister_event_listener removes a listener from its subscriber's + subscriberDirectory record, the TopicManager for the associated topic, and + the listenerDirectory. + """ + + def unregister_event_listener(self, subscriberid, subscription_name, listener_key): + listenerid = -1 + if subscriberid in self.subscriberDirectory: + if subscription_name in self.subscriberDirectory[subscriberid]: + if listener_key in self.subscriberDirectory[subscriberid][subscription_name]: + listenerid = self.subscriberDirectory[subscriberid][subscription_name][ + listener_key + ] + debug.output( + 'Unregistering listener on listener_key %s, subscription %s' + % (listener_key, subscription_name), + listenerid, + subscriberid, + ) + topic_list = self._map_listener_key_to_topic_list( + subscription_name, listener_key + ) + for topic_name in topic_list: + self.topicDirectory[topic_name].unregister_listener(listenerid) + del self.subscriberDirectory[subscriberid][subscription_name][listener_key] + debug.output( + 'Listener on listener_key %s, subscription %s unregistered' + % (listener_key, subscription_name), + listenerid, + subscriberid, + ) + else: + raise EventServiceError('Listener key not recognized.') + """ + Do not raise exception if subscriberid/subscription_name turn out to be + invalid as this can very well happen if subscriber/subscription object + is garbage collected before its listener object. In such a scenario, + unregister_subscriber/remove_subscription will clean up this listener also. + """ + return listenerid + + """ + remove_subscription is called when a subscription object is being deleted. + It unregisters associated listeners from their respective TopicManagers and the + listenerDirectory, and then removes its subscription information from the + subscriber entry in subscriberDirectory. + """ + + def remove_subscription(self, subscriberid, subscription_name): + listener_list = [] + if subscriberid in self.subscriberDirectory: + if subscription_name in self.subscriberDirectory[subscriberid]: + debug.output( + "\n\n------Subscriber's subscription to %s is being removed" + % subscription_name, + subscriberid, + ) + for listener_key in self.subscriberDirectory[subscriberid][subscription_name]: + listenerid = self.subscriberDirectory[subscriberid][subscription_name][ + listener_key + ] + debug.output( + 'Unregistering listener on listener_key %s, subscription %s' + % (listener_key, subscription_name), + listenerid, + subscriberid, + ) + topic_list = self._map_listener_key_to_topic_list( + subscription_name, listener_key + ) + for topic_name in topic_list: + self.topicDirectory[topic_name].unregister_listener(listenerid) + debug.output( + 'Listener on listener_key %s, subscription %s unregistered' + % (listener_key, subscription_name), + listenerid, + subscriberid, + ) + listener_list.append(listenerid) + del self.subscriberDirectory[subscriberid][subscription_name] + debug.output( + "Subscriber's subscription to %s removed" % subscription_name, subscriberid + ) + """ + Do not raise exception if subscriberid/subscription_name turn out to be + invalid as this can very well happen if subscriber object is garbage + collected before its subscription object. In such a scenario, + unregister_subscriber will clean up this subscription as well. + """ + return listener_list + + """""" """Subscription methods end here""" """""" + + """""" """Methods internal to the event service start here""" """""" + + """ + A listener_key may specify a bunch of topics using wildcarding. + Currently this is not supported. Need a more rigorous design to allow + wildcarding. + """ + + def _map_listener_key_to_topic_list(self, subscription_name, listener_key): + topic_list = [] + topic_list.append(listener_key) + return topic_list + + """""" """Methods internal to the event service end here""" """""" diff --git a/ipsframework/event_service_proxy.py b/ipsframework/event_service_proxy.py new file mode 100644 index 00000000..846faf20 --- /dev/null +++ b/ipsframework/event_service_proxy.py @@ -0,0 +1,209 @@ +# ------------------------------------------------------------------------------- +# Copyright 2006-2022 UT-Battelle, LLC. See LICENSE for more information. +# ------------------------------------------------------------------------------- + + +class EventServiceProxy: + def get_topic(self, topic_name): + pass + + def exists_topic(self, topic_name): + pass + + def register_subscriber(self): + pass + + def unregister_subscriber(self, subscriberid): + pass + + def get_subscription(self, subscriberid, subscription_name): + pass + + def process_events(self, subscriberid): + pass + + def send_event(self, topic_name, event_name, event_body): + pass + + def create_listener(self): + pass + + def register_event_listener( + self, subscriberid, subscription_name, listener_key, listenerid, ref_listener + ): + pass + + def unregister_event_listener(self, subscriberid, subscription_name, listener_key): + pass + + def remove_subscription(self, subscriberid, subscription_name): + pass + + +# TODO: Is eventService.py the right placeholder for this class?# +class EventServiceFwkProxy(EventServiceProxy): + def __init__(self, event_service): + self.event_service = event_service + self.listenerDirectory = {} + + def get_topic(self, topic_name): + return self.event_service.get_topic(topic_name) + + def exists_topic(self, topic_name): + return self.event_service.exists_topic(topic_name) + + def register_subscriber(self): + return self.event_service.register_subscriber() + + def unregister_subscriber(self, subscriberid): + listener_list = self.event_service.unregister_subscriber(subscriberid) + for listenerid in listener_list: + self._remove_event_listener(listenerid) + + def get_subscription(self, subscriberid, subscription_name): + self.event_service.get_subscription(subscriberid, subscription_name) + + def process_events(self, subscriberid): + event_list = self.event_service.process_events(subscriberid) + for listenerid in event_list: + for topic_name in event_list[listenerid]: + for the_event in event_list[listenerid][topic_name]: + self.listenerDirectory[listenerid].process_event(topic_name, the_event) + + def send_event(self, topic_name, event_name, event_body): + self.event_service.send_event(topic_name, event_name, event_body) + + def create_listener(self): + return self.event_service.create_listener() + + def register_event_listener( + self, subscriberid, subscription_name, listener_key, listenerid, ref_listener + ): + self.event_service.register_event_listener( + subscriberid, subscription_name, listener_key, listenerid + ) + self._add_event_listener(listenerid, ref_listener) + + def unregister_event_listener(self, subscriberid, subscription_name, listener_key): + listenerid = self.event_service.unregister_event_listener( + subscriberid, subscription_name, listener_key + ) + self._remove_event_listener(listenerid) + + def remove_subscription(self, subscriberid, subscription_name): + listener_list = self.event_service.remove_subscription(subscriberid, subscription_name) + for listenerid in listener_list: + self._remove_event_listener(listenerid) + + def _add_event_listener(self, listenerid, ref_listener): + if listenerid not in self.listenerDirectory: + self.listenerDirectory[listenerid] = ref_listener + + def _remove_event_listener(self, listenerid): + if listenerid in self.listenerDirectory: + del self.listenerDirectory[listenerid] + + +# TODO: Is services.py the right placeholder for this class?# +class EventServiceCmpProxy(EventServiceProxy): + def __init__(self, service_proxy): + self.service_proxy = service_proxy + self.listenerDirectory = {} + + def get_topic(self, topic_name): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'get_topic', topic_name + ) + return self.service_proxy._get_service_response(msg_id, True) + + def exists_topic(self, topic_name): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'exists_topic', topic_name + ) + return self.service_proxy._get_service_response(msg_id, True) + + def register_subscriber(self): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'register_subscriber' + ) + return self.service_proxy._get_service_response(msg_id, True) + + def unregister_subscriber(self, subscriberid): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'unregister_subscriber', subscriberid + ) + listener_list = self.service_proxy._get_service_response(msg_id, True) + for listenerid in listener_list: + self._remove_event_listener(listenerid) + + def get_subscription(self, subscriberid, subscription_name): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'get_subscription', subscriberid, subscription_name + ) + self.service_proxy._get_service_response(msg_id, True) + + def process_events(self, subscriberid): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'process_events', subscriberid + ) + event_list = self.service_proxy._get_service_response(msg_id, True) + for listenerid in event_list: + for topic_name in event_list[listenerid]: + for the_event in event_list[listenerid][topic_name]: + self.listenerDirectory[listenerid].process_event(topic_name, the_event) + + def send_event(self, topic_name, event_name, event_body): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'send_event', topic_name, event_name, event_body + ) + self.service_proxy._get_service_response(msg_id, True) + + def create_listener(self): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, 'create_listener' + ) + return self.service_proxy._get_service_response(msg_id, True) + + def register_event_listener( + self, subscriberid, subscription_name, listener_key, listenerid, ref_listener + ): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, + 'register_event_listener', + subscriberid, + subscription_name, + listener_key, + listenerid, + ) + self.service_proxy._get_service_response(msg_id, True) + self._add_event_listener(listenerid, ref_listener) + + def unregister_event_listener(self, subscriberid, subscription_name, listener_key): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, + 'unregister_event_listener', + subscriberid, + subscription_name, + listener_key, + ) + listenerid = self.service_proxy._get_service_response(msg_id, True) + self._remove_event_listener(listenerid) + + def remove_subscription(self, subscriberid, subscription_name): + msg_id = self.service_proxy._invoke_service( + self.service_proxy.fwk.component_id, + 'remove_subscription', + subscriberid, + subscription_name, + ) + listener_list = self.service_proxy._get_service_response(msg_id, True) + for listenerid in listener_list: + self._remove_event_listener(listenerid) + + def _add_event_listener(self, listenerid, ref_listener): + if listenerid not in self.listenerDirectory: + self.listenerDirectory[listenerid] = ref_listener + + def _remove_event_listener(self, listenerid): + if listenerid in self.listenerDirectory: + del self.listenerDirectory[listenerid] diff --git a/ipsframework/ips.py b/ipsframework/ips.py index bac399d2..0aed6638 100755 --- a/ipsframework/ips.py +++ b/ipsframework/ips.py @@ -62,21 +62,27 @@ import socket import sys import time -from typing import Callable, Dict, Iterable, List, Optional +from collections.abc import Callable, Iterable from ipsframework import platformspec from ipsframework._version import get_versions from ipsframework.cca_es_spec import initialize_event_service -from ipsframework.componentRegistry import ComponentID, ComponentRegistry -from ipsframework.configurationManager import ConfigurationManager -from ipsframework.dataManager import DataManager -from ipsframework.eventService import EventService -from ipsframework.ips_es_spec import eventManager -from ipsframework.ipsExceptions import BlockedMessageException -from ipsframework.ipsutil import getTimeString -from ipsframework.messages import Message, MethodInvokeMessage, MethodResultMessage, ServiceRequestMessage, ServiceResponseMessage -from ipsframework.resourceManager import ResourceManager -from ipsframework.taskManager import TaskManager +from ipsframework.component_registry import ComponentID, ComponentRegistry +from ipsframework.configuration_manager import ConfigurationManager +from ipsframework.data_manager import DataManager +from ipsframework.event_service import EventService +from ipsframework.ips_es_spec import EventManager +from ipsframework.ips_exceptions import BlockedMessageError +from ipsframework.ipsutil import get_time_string +from ipsframework.messages import ( + Message, + MethodInvokeMessage, + MethodResultMessage, + ServiceRequestMessage, + ServiceResponseMessage, +) +from ipsframework.resource_manager import ResourceManager +from ipsframework.task_manager import TaskManager if sys.version_info[0] != 3 or sys.version_info[1] < 9: print('IPS is only compatible with Python 3.9 or higher', file=sys.stderr) @@ -129,9 +135,9 @@ class Framework: def __init__( self, - config_file_list: List[str], + config_file_list: list[str], log_file_name: str, - platform_file_name: Optional[str] = None, + platform_file_name: str | None = None, debug: bool = False, verbose_debug: bool = False, cmd_nodes: int = 0, @@ -155,10 +161,12 @@ def __init__( # reference to this class's component ID self.component_id = ComponentID(self.__class__.__name__, 'FRAMEWORK') # map of ports - self.port_map: Dict[int, str] = {} + self.port_map: dict[int, str] = {} current_dir = inspect.getfile(inspect.currentframe()) - (self.platform_file_name, self.ipsShareDir) = platformspec.get_share_and_platform(platform_file_name, current_dir) + (self.platform_file_name, self.ipsShareDir) = platformspec.get_share_and_platform( + platform_file_name, current_dir + ) # config file list self.config_file_list = config_file_list @@ -170,8 +178,10 @@ def __init__( self.start_time = self.cur_time self.event_service = EventService(self) initialize_event_service(self.event_service) - self.event_manager = eventManager(self) - self.config_manager = ConfigurationManager(self, self.config_file_list, self.platform_file_name) + self.event_manager = EventManager(self) + self.config_manager = ConfigurationManager( + self, self.config_file_list, self.platform_file_name + ) self.resource_manager = ResourceManager(self) self.data_manager = DataManager(self) self.task_manager = TaskManager(self) @@ -199,9 +209,19 @@ def __init__( try: # each manager should create their own event manager if they # want to send and receive events - self.config_manager.initialize(self.data_manager, self.resource_manager, self.task_manager) - self.task_manager.initialize(self.data_manager, self.resource_manager, self.config_manager) - self.resource_manager.initialize(self.data_manager, self.task_manager, self.config_manager, cmd_nodes, cmd_ppn) + self.config_manager.initialize( + self.data_manager, self.resource_manager, self.task_manager + ) + self.task_manager.initialize( + self.data_manager, self.resource_manager, self.config_manager + ) + self.resource_manager.initialize( + self.data_manager, + self.task_manager, + self.config_manager, + cmd_nodes, + cmd_ppn, + ) except Exception: self.exception('Problem initializing managers') self.terminate_all_sims(status=Message.FAILURE) @@ -209,7 +229,7 @@ def __init__( self.blocked_messages = [] # SIMYAN: determine the sim_root for the Framework to use later fwk_comps = self.config_manager.get_framework_components() - main_fwk_comp = self.comp_registry.getEntry(fwk_comps[0]) + main_fwk_comp = self.comp_registry.get_entry(fwk_comps[0]) self.sim_root = os.path.abspath(main_fwk_comp.services.get_config_param('SIM_ROOT')) def get_inq(self): @@ -219,7 +239,11 @@ def get_inq(self): """ return self.in_queue - def register_service_handler(self, service_list: Iterable[str], handler: Callable[[ServiceRequestMessage], None]): + def register_service_handler( + self, + service_list: Iterable[str], + handler: Callable[[ServiceRequestMessage], None], + ): """ Register a call back method to handle a list of framework service invocations. @@ -241,34 +265,54 @@ def _dispatch_service_request(self, msg: ServiceRequestMessage): method is conveyed to the caller along the appropriate queue in a :class:`messages.ServiceResponseMessage`. All exceptions are passed on to the caller, except for the - :class:`ipsExceptions.BlockedMessageException`, which causes the + :class:`ipsExceptions.BlockedMessageError`, which causes the message to be blocked until the request can be satisfied. """ method_name = msg.target_method comp_id = msg.sender_id - self.debug('Framework dispatching method: %s from %s', method_name, str(comp_id)) + self.debug( + 'Framework dispatching method: %s from %s', + method_name, + str(comp_id), + ) try: handler = self.service_handler[method_name] except KeyError: self.exception('Unsupported method : %s', method_name) response_msg = ServiceResponseMessage( - self.component_id, comp_id, msg.message_id, Message.FAILURE, Exception('Unsupported method : %s' % (method_name)) + self.component_id, + comp_id, + msg.message_id, + Message.FAILURE, + Exception('Unsupported method : %s' % (method_name)), ) else: try: ret_val = handler(msg) - except BlockedMessageException as e: + except BlockedMessageError as e: if self.verbose_debug: self.debug('Blocked message : %s', str(e)) self.blocked_messages.append(msg) return except Exception as e: # self.exception('Exception handling service message: %s - %s', str(msg.__dict__), str(e)) - response_msg = ServiceResponseMessage(self.component_id, comp_id, msg.message_id, Message.FAILURE, e) + response_msg = ServiceResponseMessage( + self.component_id, + comp_id, + msg.message_id, + Message.FAILURE, + e, + ) else: - response_msg = ServiceResponseMessage(self.component_id, comp_id, msg.message_id, Message.SUCCESS, ret_val) - - response_q = self.comp_registry.getComponentArtifact(comp_id, 'svc_response_q') + response_msg = ServiceResponseMessage( + self.component_id, + comp_id, + msg.message_id, + Message.SUCCESS, + ret_val, + ) + + response_q = self.comp_registry.get_component_artifact(comp_id, 'svc_response_q') response_q.put(response_msg) def log(self, msg: object, *args): @@ -324,7 +368,14 @@ def _invoke_framework_comps(self, fwk_comps: Iterable[ComponentID], method_name: outstanding_fwk_calls = [] for comp_id in fwk_comps: - msg = ServiceRequestMessage(self.component_id, self.component_id, comp_id, 'init_call', method_name, 0) + msg = ServiceRequestMessage( + self.component_id, + self.component_id, + comp_id, + 'init_call', + method_name, + 0, + ) self.debug('Framework sending message %s ', msg.__dict__) call_id = self.task_manager.init_call(msg, manage_return=False) outstanding_fwk_calls.append(call_id) @@ -351,7 +402,10 @@ def _invoke_framework_comps(self, fwk_comps: Iterable[ComponentID], method_name: raise msg.args[0] outstanding_fwk_calls.remove(msg.call_id) else: - self.error('Framework received unexpected message : %s', str(msg.__dict__)) + self.error( + 'Framework received unexpected message : %s', + str(msg.__dict__), + ) def run(self) -> bool: """ @@ -400,7 +454,7 @@ def run(self) -> bool: # SIMYAN: get the runspaceInit_component and invoke its init() method # this creates the base directory and container file for the simulation # and copies the conf files into both and change directory to base dir - main_fwk_comp = self.comp_registry.getEntry(fwk_comps[0]) + main_fwk_comp = self.comp_registry.get_entry(fwk_comps[0]) self.sim_root = os.path.abspath(main_fwk_comp.services.get_config_param('SIM_ROOT')) self._invoke_framework_comps(fwk_comps, 'init') @@ -410,21 +464,31 @@ def run(self) -> bool: for comp_id in fwk_comps: msg_list = [] for method in ['step', 'finalize']: - req_msg = ServiceRequestMessage(self.component_id, self.component_id, comp_id, 'init_call', method, 0) + req_msg = ServiceRequestMessage( + self.component_id, + self.component_id, + comp_id, + 'init_call', + method, + 0, + ) msg_list.append((req_msg, None, str(comp_id), method, 0)) outstanding_sim_calls[str(comp_id)] = msg_list # generate a queue of invocation messages for each simulation # - list will look like: [init_comp.init(), init_comp.step(), init_comp.finalize(), - # driver.init(), driver.step(), driver.finalize()] + # Driver.init(), Driver.step(), Driver.finalize()] # these messages will be sent on a FIFO basis, thus running the init components, # then the corresponding drivers. for sim_name, comp_list in sim_comps.items(): msg_list = [] self._send_monitor_event(sim_name, 'IPS_START', 'Starting IPS Simulation') self._send_dynamic_sim_event(sim_name=sim_name, event_type='IPS_START') - comment = 'Nodes = %d PPN = %d' % (self.resource_manager.num_nodes, self.resource_manager.ppn) + comment = 'Nodes = %d PPN = %d' % ( + self.resource_manager.num_nodes, + self.resource_manager.ppn, + ) self._send_monitor_event(sim_name, 'IPS_RESOURCE_ALLOC', comment) # SIMYAN: ordered list of methods to call methods = ['init', 'step', 'finalize'] @@ -432,7 +496,14 @@ def run(self) -> bool: # SIMYAN: add each method call to the msg_list for comp_id in comp_list: for method in methods: - req_msg = ServiceRequestMessage(self.component_id, self.component_id, comp_id, 'init_call', method, 0) + req_msg = ServiceRequestMessage( + self.component_id, + self.component_id, + comp_id, + 'init_call', + method, + 0, + ) msg_list.append((req_msg, sim_name, str(comp_id), method, 0)) # SIMYAN: add the msg_list to the outstanding sim calls if msg_list: @@ -449,12 +520,24 @@ def run(self) -> bool: msg, sim_name, comp, method, arg = msg_list.pop(0) # noqa: PLW2901 (TODO: make sure this is intended behavior, change variable name if it is and refactor if not) self.debug('Framework sending message %s ', msg.__dict__) if sim_name is not None: - self._send_monitor_event(sim_name=sim_name, comment=f'Target = {comp}:{method}({arg})', eventType='IPS_CALL_BEGIN') + self._send_monitor_event( + sim_name=sim_name, + comment=f'Target = {comp}:{method}({arg})', + event_type='IPS_CALL_BEGIN', + ) call_id = self.task_manager.init_call(msg, manage_return=False) self.call_queue_map[call_id] = msg_list - self.outstanding_calls_list[call_id] = sim_name, comp, method, arg, time.time() + self.outstanding_calls_list[call_id] = ( + sim_name, + comp, + method, + arg, + time.time(), + ) except Exception: - self.exception('encountered exception during fwk.run() sending first round of invocations (init of inits and fwk comps)') + self.exception( + 'encountered exception during fwk.run() sending first round of invocations (init of inits and fwk comps)' + ) self.terminate_all_sims(status=Message.FAILURE) raise @@ -491,12 +574,14 @@ def run(self) -> bool: self.task_manager.return_call(msg) continue # Message is a result from a framework invocation - sim_name, comp, method, arg, start_time = self.outstanding_calls_list.pop(msg.call_id) + sim_name, comp, method, arg, start_time = self.outstanding_calls_list.pop( + msg.call_id + ) if sim_name is not None: self._send_monitor_event( sim_name=sim_name, comment=f'Target = {comp}:{method}({arg})', - eventType='IPS_CALL_END', + event_type='IPS_CALL_END', start_time=start_time, end_time=time.time(), target=comp, @@ -506,7 +591,11 @@ def run(self) -> bool: sim_msg_list = self.call_queue_map[msg.call_id] del self.call_queue_map[msg.call_id] if msg.status == Message.FAILURE: - self.error('received a failure message from component %s : %s', msg.sender_id, str(msg.args)) + self.error( + 'received a failure message from component %s : %s', + msg.sender_id, + str(msg.args), + ) # No need to process remaining messages for this simulation sim_msg_list = [] comment = 'Simulation Execution Error' @@ -520,12 +609,24 @@ def run(self) -> bool: try: next_call_msg, sim_name, comp, method, arg = sim_msg_list.pop(0) if sim_name is not None: - self._send_monitor_event(sim_name=sim_name, comment=f'Target = {comp}:{method}({arg})', eventType='IPS_CALL_BEGIN') + self._send_monitor_event( + sim_name=sim_name, + comment=f'Target = {comp}:{method}({arg})', + event_type='IPS_CALL_BEGIN', + ) call_id = self.task_manager.init_call(next_call_msg, manage_return=False) - self.outstanding_calls_list[call_id] = sim_name, comp, method, arg, time.time() + self.outstanding_calls_list[call_id] = ( + sim_name, + comp, + method, + arg, + time.time(), + ) self.call_queue_map[call_id] = sim_msg_list except IndexError: - sim_comps = self.config_manager.get_component_map() # Get any new dynamic simulations + sim_comps = ( + self.config_manager.get_component_map() + ) # Get any new dynamic simulations if sim_name in sim_comps: self._send_monitor_event(sim_name, 'IPS_END', comment, ok) self._send_dynamic_sim_event(sim_name, 'IPS_END', ok) @@ -549,7 +650,14 @@ def initiate_new_simulation(self, sim_name: str): self._send_dynamic_sim_event(sim_name=sim_name, event_type='IPS_START') for comp_id in comp_list: for method in ['init', 'step', 'finalize']: - req_msg = ServiceRequestMessage(self.component_id, self.component_id, comp_id, 'init_call', method, 0) + req_msg = ServiceRequestMessage( + self.component_id, + self.component_id, + comp_id, + 'init_call', + method, + 0, + ) msg_list.append((req_msg, sim_name, str(comp_id), method, 0)) # send off first round of invocations... @@ -557,9 +665,26 @@ def initiate_new_simulation(self, sim_name: str): self.debug('Framework sending message %s ', msg.__dict__) call_id = self.task_manager.init_call(msg, manage_return=False) self.call_queue_map[call_id] = msg_list - self.outstanding_calls_list[call_id] = sim_name, comp, method, arg, time.time() - - def _send_monitor_event(self, sim_name='', eventType='', comment='', ok=True, target=None, operation=None, start_time=None, end_time=None, call_id=0): + self.outstanding_calls_list[call_id] = ( + sim_name, + comp, + method, + arg, + time.time(), + ) + + def _send_monitor_event( + self, + sim_name='', + event_type='', + comment='', + ok=True, + target=None, + operation=None, + start_time=None, + end_time=None, + call_id=0, + ): """ Publish a monitor event to the *_IPS_MONITOR* event topic. Event topics that start with an underscore are reserved for use by the @@ -570,7 +695,7 @@ def _send_monitor_event(self, sim_name='', eventType='', comment='', ok=True, ta When a single message is published, ALL of these components will handle the published message (there is no "shared queue"). :param sim_name: The name of the simulation to which this even belongs. - :param eventType: The type of the event. + :param event_type: The type of the event. :param comment: A string containing comment that describes the event. :param ok: A string containing the values 'True' or 'False', based on whether the event indicates normal simulation execution, or an @@ -579,26 +704,31 @@ def _send_monitor_event(self, sim_name='', eventType='', comment='', ok=True, ta """ event_time = time.time() if self.verbose_debug: - self.debug('_send_monitor_event(%s - %s)', sim_name, eventType) + self.debug('_send_monitor_event(%s - %s)', sim_name, event_type) portal_data = {} portal_data['code'] = 'Framework' # eventData['portal_runid'] = self.portalRunId - portal_data['eventtype'] = eventType + portal_data['eventtype'] = event_type portal_data['ok'] = ok portal_data['comment'] = comment - portal_data['walltime'] = '%.2f' % (event_time - self.config_manager.sim_map[sim_name].start_time) - portal_data['time'] = getTimeString(time.localtime(event_time)) + portal_data['walltime'] = '%.2f' % ( + event_time - self.config_manager.sim_map[sim_name].start_time + ) + portal_data['time'] = get_time_string(time.localtime(event_time)) # portal_data['phystimestamp'] = self.timeStamp get_config = self.config_manager.get_config_parameter - if eventType == 'IPS_START': + if event_type == 'IPS_START': # The 'IPS_START' event is always the first event sent from a component, and it should always be submitted internally. # This event will always mark the first time a component has registered, # and sending this event is indicative of the first time we make a Web Portal call and the first time we write to the event log files. user = self.config_manager.get_platform_parameter('USER') host = self.config_manager.get_platform_parameter('HOST') d = datetime.datetime.now() - date_str = '%s.%03d' % (d.strftime('%Y-%m-%dT%H:%M:%S'), int(d.microsecond / 1000)) + date_str = '%s.%03d' % ( + d.strftime('%Y-%m-%dT%H:%M:%S'), + int(d.microsecond / 1000), + ) portal_runid = f'{sim_name}_{host}_{user}_{date_str}' portal_data['state'] = 'Running' @@ -642,7 +772,9 @@ def _send_monitor_event(self, sim_name='', eventType='', comment='', ok=True, ta portal_data['sim_runid'] = get_config(sim_name, 'RUN_ID') except KeyError: pass - portal_data['startat'] = getTimeString(time.localtime(self.config_manager.sim_map[sim_name].start_time)) + portal_data['startat'] = get_time_string( + time.localtime(self.config_manager.sim_map[sim_name].start_time) + ) portal_data['ips_version'] = get_versions()['version'] portal_data['portal_runid'] = portal_runid @@ -651,19 +783,21 @@ def _send_monitor_event(self, sim_name='', eventType='', comment='', ok=True, ta except KeyError: pass - elif eventType == 'IPS_END': + elif event_type == 'IPS_END': # The IPS_END event is always the last event called by the framework, ONLY sent out to indicate that there are no remaining messages to handle. portal_data['state'] = 'Completed' - portal_data['stopat'] = getTimeString(time.localtime(event_time)) + portal_data['stopat'] = get_time_string(time.localtime(event_time)) # Zipkin json format portal_data['trace'] = { 'timestamp': int(self.config_manager.sim_map[sim_name].start_time * 1e6), - 'duration': int((event_time - self.config_manager.sim_map[sim_name].start_time) * 1e6), + 'duration': int( + (event_time - self.config_manager.sim_map[sim_name].start_time) * 1e6 + ), 'localEndpoint': {'serviceName': f'{sim_name}@{self.component_id}'}, 'id': hashlib.md5(f'{sim_name}@{self.component_id}'.encode()).hexdigest()[:16], 'tags': {'total_cores': str(self.resource_manager.total_cores)}, } - elif eventType == 'IPS_CALL_END': + elif event_type == 'IPS_CALL_END': # The IPS_CALL_END event is always the last event called by the framework, trace = {} # Zipkin json format if start_time is not None and end_time is not None: @@ -672,8 +806,12 @@ def _send_monitor_event(self, sim_name='', eventType='', comment='', ok=True, ta if target is not None: trace['localEndpoint'] = {'serviceName': target} trace['name'] = operation - trace['id'] = hashlib.md5(f'{target}:{operation}:{call_id}'.encode()).hexdigest()[:16] - trace['parentId'] = hashlib.md5(f'{sim_name}@{self.component_id}'.encode()).hexdigest()[:16] + trace['id'] = hashlib.md5(f'{target}:{operation}:{call_id}'.encode()).hexdigest()[ + :16 + ] + trace['parentId'] = hashlib.md5( + f'{sim_name}@{self.component_id}'.encode() + ).hexdigest()[:16] if trace: portal_data['trace'] = trace @@ -686,7 +824,11 @@ def _send_monitor_event(self, sim_name='', eventType='', comment='', ok=True, ta if self.verbose_debug: self.debug('Publishing %s', str(event_body)) # this message will be published to any component subscribed to '_IPS_MONITOR' - this generally includes the local logger bridge and the portal bridge - self.event_manager.publish(topicName='_IPS_MONITOR', eventName='IPS_SIM', eventBody=event_body) + self.event_manager.publish( + topic_name='_IPS_MONITOR', + event_name='IPS_SIM', + event_body=event_body, + ) def _send_dynamic_sim_event(self, sim_name='', event_type='', ok=True): self.debug('_send_dynamic_sim_event(%s:%s)', event_type, sim_name) @@ -695,7 +837,11 @@ def _send_dynamic_sim_event(self, sim_name='', event_type='', ok=True): event_data['SIM_NAME'] = sim_name event_data['ok'] = ok self.debug('Publishing %s', str(event_data)) - self.event_manager.publish(topicName='_IPS_DYNAMIC_SIMULATION', eventName='IPS_DYNAMIC_SIM', eventBody=event_data) + self.event_manager.publish( + topic_name='_IPS_DYNAMIC_SIMULATION', + event_name='IPS_DYNAMIC_SIM', + event_body=event_data, + ) # TODO mark status as a "Literal" if we move to Python >= 3.8 def send_terminate_msg(self, sim_name: str, status=Message.SUCCESS): @@ -713,7 +859,7 @@ def send_terminate_msg(self, sim_name: str, status=Message.SUCCESS): comp_ids = self.comp_registry.get_component_ids(sim_name) for comp_id in comp_ids: try: - invocation_q = self.comp_registry.getComponentArtifact(comp_id, 'invocation_q') + invocation_q = self.comp_registry.get_component_artifact(comp_id, 'invocation_q') call_id = self.task_manager.get_call_id() msg = MethodInvokeMessage(self.component_id, comp_id, call_id, 'terminate', status) self.debug('Sending terminate message to %s', str(comp_id)) @@ -756,23 +902,80 @@ def main(): print('IPS using platform file :', platform_default, file=sys.stderr) parser = argparse.ArgumentParser() - parser.add_argument('--version', action='version', version='%(prog)s ' + get_versions()['version']) - parser.add_argument('--simulation', '-i', '--config', '-j', required=True, help='IPS simulation/config file') parser.add_argument( - '--platform', '-p', dest='platform_filename', default=platform_default, required=not platform_default, help='IPS platform configuration file' + '--version', + action='version', + version='%(prog)s ' + get_versions()['version'], + ) + parser.add_argument( + '--simulation', + '-i', + '--config', + '-j', + required=True, + help='IPS simulation/config file', + ) + parser.add_argument( + '--platform', + '-p', + dest='platform_filename', + default=platform_default, + required=not platform_default, + help='IPS platform configuration file', + ) + parser.add_argument( + '--debug', + '-d', + default=False, + action='store_true', + help='Turn on debugging', + ) + parser.add_argument( + '--verbose', + '-v', + dest='verbose_debug', + default=False, + action='store_true', + help='Run IPS verbosely', + ) + parser.add_argument( + '--log', + '-l', + dest='log_file', + default='sys.stdout', + help='IPS Log file', + ) + parser.add_argument( + '--nodes', + '-n', + dest='cmd_nodes', + default='0', + type=int, + help='Computer nodes', + ) + parser.add_argument( + '--ppn', + '-o', + dest='cmd_ppn', + default='0', + type=int, + help='Computer processor per nodes', ) - parser.add_argument('--debug', '-d', default=False, action='store_true', help='Turn on debugging') - parser.add_argument('--verbose', '-v', dest='verbose_debug', default=False, action='store_true', help='Run IPS verbosely') - parser.add_argument('--log', '-l', dest='log_file', default='sys.stdout', help='IPS Log file') - parser.add_argument('--nodes', '-n', dest='cmd_nodes', default='0', type=int, help='Computer nodes') - parser.add_argument('--ppn', '-o', dest='cmd_ppn', default='0', type=int, help='Computer processor per nodes') options = parser.parse_args() - cfgFile_list = options.simulation.split(',') + cfg_file_list = options.simulation.split(',') try: - fwk = Framework(cfgFile_list, options.log_file, options.platform_filename, options.debug, options.verbose_debug, options.cmd_nodes, options.cmd_ppn) + fwk = Framework( + cfg_file_list, + options.log_file, + options.platform_filename, + options.debug, + options.verbose_debug, + options.cmd_nodes, + options.cmd_ppn, + ) fwk.run() except Exception as e: print(e, file=sys.stderr) diff --git a/ipsframework/ips_dakota_client.py b/ipsframework/ips_dakota_client.py index 2b41781c..8b660f09 100755 --- a/ipsframework/ips_dakota_client.py +++ b/ipsframework/ips_dakota_client.py @@ -31,16 +31,20 @@ def run(self): """ # parse file try: - self.platform_conf = ConfigObj(self.platform_fname, interpolation='template', file_error=True) - except (IOError, SyntaxError): + self.platform_conf = ConfigObj( + self.platform_fname, interpolation='template', file_error=True + ) + except (OSError, SyntaxError): raise """ Master Config file """ # parse file try: - self.old_master_conf = ConfigObj(self.config_file, interpolation='template', file_error=True) - except (IOError, SyntaxError): + self.old_master_conf = ConfigObj( + self.config_file, interpolation='template', file_error=True + ) + except (OSError, SyntaxError): raise # Import environment variables into config file # giving precedence to config file definitions in case of duplicates @@ -84,7 +88,15 @@ def run(self): try: conn = Client(str(server_address), 'AF_UNIX') except Exception: - print('%s: %d Failed to connect to %s: %s' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), trials, server_address, str(sys.argv))) + print( + '%s: %d Failed to connect to %s: %s' + % ( + time.strftime('%b %d %Y %H:%M:%S', time.localtime()), + trials, + server_address, + str(sys.argv), + ) + ) sys.stdout.flush() if trials == num_trials - 1: raise @@ -115,7 +127,9 @@ def main(argv=None): config_file = os.environ['IPS_DAKOTA_config'] try: - ips_executer = IPSDakotaClient(config_file, log_file_name, platform_filename, debug, in_file, out_file) + ips_executer = IPSDakotaClient( + config_file, log_file_name, platform_filename, debug, in_file, out_file + ) ips_executer.run() except Exception: raise diff --git a/ipsframework/ips_dakota_dynamic.py b/ipsframework/ips_dakota_dynamic.py index 3f197dc2..a598affb 100755 --- a/ipsframework/ips_dakota_dynamic.py +++ b/ipsframework/ips_dakota_dynamic.py @@ -20,7 +20,9 @@ class DakotaDynamic: - def __init__(self, dakota_cfg, log_file, platform_filename, debug, ips_config_template, restart_file): + def __init__( + self, dakota_cfg, log_file, platform_filename, debug, ips_config_template, restart_file + ): self.dakota_cfg = dakota_cfg self.log_file = log_file self.platform_fname = platform_filename @@ -40,7 +42,7 @@ def run(self): Control variables expected in the format COMPONENT__VARIABLE (two _) """ try: - self.dakota_conf = [t.strip() for t in open(self.dakota_cfg).readlines()] + self.dakota_conf = [t.strip() for t in open(self.dakota_cfg)] except Exception: raise @@ -50,22 +52,30 @@ def run(self): # parse file try: current_dir = inspect.getfile(inspect.currentframe()) - (self.platform_fname, ipsShareDir) = platformspec.get_share_and_platform(self.platform_fname, current_dir) - - if ipsShareDir: - haveComp = False - if os.path.exists(os.path.join(ipsShareDir, 'component-generic.conf')): - comp_conf_file = os.path.join(ipsShareDir, 'component-generic.conf') - comp_confgobj = ConfigObj(comp_conf_file, interpolation='template', file_error=True) - haveComp = True - - self.platform_conf = ConfigObj(self.platform_fname, interpolation='template', file_error=True) - if haveComp: + (self.platform_fname, ips_share_dir) = platformspec.get_share_and_platform( + self.platform_fname, current_dir + ) + + if ips_share_dir: + have_comp = False + if os.path.exists(os.path.join(ips_share_dir, 'component-generic.conf')): + comp_conf_file = os.path.join(ips_share_dir, 'component-generic.conf') + comp_confgobj = ConfigObj( + comp_conf_file, interpolation='template', file_error=True + ) + have_comp = True + + self.platform_conf = ConfigObj( + self.platform_fname, interpolation='template', file_error=True + ) + if have_comp: self.platform_conf.merge(comp_confgobj) else: - self.platform_conf = ConfigObj(self.platform_fname, interpolation='template', file_error=True) + self.platform_conf = ConfigObj( + self.platform_fname, interpolation='template', file_error=True + ) - except (IOError, SyntaxError): + except (OSError, SyntaxError): raise """ @@ -73,8 +83,10 @@ def run(self): """ # parse file try: - self.template_conf = ConfigObj(self.config_template, interpolation='template', file_error=True) - except (IOError, SyntaxError): + self.template_conf = ConfigObj( + self.config_template, interpolation='template', file_error=True + ) + except (OSError, SyntaxError): raise for k, v in self.platform_conf.items(): if k not in self.template_conf: @@ -108,11 +120,13 @@ def run(self): prog = raw_prog.strip(' "\'') exec_prog = which(prog) if not exec_prog: - raise Exception('Error: analysis driver %s not found in path' % prog) + raise Exception('Error: analysis Driver %s not found in path' % prog) line.replace(prog, exec_prog) elif tokens[0] == 'system': if 'asynchronous' not in line: - raise Exception('Asynchronous specification missing from DAKOTA system line in interface section') + raise Exception( + 'Asynchronous specification missing from DAKOTA system line in interface section' + ) match = re.search(r'evaluation_concurrency\s*=\s*\d*', line) if match: conc_tokens = match.group(0).split(' =') @@ -151,7 +165,10 @@ def run(self): try: os.makedirs(sim_root, exist_ok=True) except OSError as oserr: - print('Error creating Simulation directory %s : %d %s' % (sim_root, oserr.errno, oserr.strerror)) + print( + 'Error creating Simulation directory %s : %d %s' + % (sim_root, oserr.errno, oserr.strerror) + ) raise for k, v in self.template_conf.items(): @@ -182,7 +199,11 @@ def run(self): if not os.path.isfile(self.restart_file): raise Exception('Error accessing DAKOTA restart file %s' % (self.restart_file)) - cmd = '%s --simulation=%s --platform=%s --verbose' % (ips, self.master_conf.filename, os.environ['IPS_DAKOTA_platform']) + cmd = '%s --simulation=%s --platform=%s --verbose' % ( + ips, + self.master_conf.filename, + os.environ['IPS_DAKOTA_platform'], + ) if self.log_file: cmd += ' --log=' + self.log_file @@ -202,7 +223,8 @@ def run(self): response = conn.recv() except Exception as inst: print( - '%s %d ips_dakota_dynamic connecting to IPS dakota bridge' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), trials), + '%s %d ips_dakota_dynamic connecting to IPS dakota bridge' + % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), trials), type(inst), str(inst), ) @@ -213,7 +235,11 @@ def run(self): else: time.sleep(5) else: - print('%s ips_dakota_dynamic received response from IPS ' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime())), str(response)) + print( + '%s ips_dakota_dynamic received response from IPS ' + % (time.strftime('%b %d %Y %H:%M:%S', time.localtime())), + str(response), + ) conn.close() break @@ -236,7 +262,8 @@ def run(self): response = conn.recv() except Exception as inst: print( - '%s %d ips_dakota_dynamic connecting to IPS dakota bridge' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), trials), + '%s %d ips_dakota_dynamic connecting to IPS dakota bridge' + % (time.strftime('%b %d %Y %H:%M:%S', time.localtime()), trials), type(inst), str(inst), ) @@ -247,14 +274,18 @@ def run(self): else: time.sleep(5) else: - print('%s ips_dakota_dynamic received response from IPS ' % (time.strftime('%b %d %Y %H:%M:%S', time.localtime())), str(response)) + print( + '%s ips_dakota_dynamic received response from IPS ' + % (time.strftime('%b %d %Y %H:%M:%S', time.localtime())), + str(response), + ) conn.close() break ips_server_proc.wait() -def printUsageMessage(): +def print_usage_message(): print( 'Usage: ips_dakota_dynamic --dakotaconfig=DAKOTA_CONFIG_FILE --simulation=CONFIG_FILE_NAME ' '--platform=PLATFORM_FILE_NAME --log=LOG_FILE_NAME --restart=DAKOTA_RESTART_FILE [--debug]' @@ -272,10 +303,14 @@ def main(argv=None): first_arg = 0 try: - opts, _ = getopt.gnu_getopt(argv[first_arg:], '', ['dakotaconfig=', 'simulation=', 'platform=', 'log=', 'restart=', 'debug']) + opts, _ = getopt.gnu_getopt( + argv[first_arg:], + '', + ['dakotaconfig=', 'simulation=', 'platform=', 'log=', 'restart=', 'debug'], + ) except getopt.error as msg: print('Invalid command line arguments', msg) - printUsageMessage() + print_usage_message() return 1 debug = False log_file_name = None @@ -297,10 +332,12 @@ def main(argv=None): debug = True if not ips_config_file or not dakota_cfg: - printUsageMessage() + print_usage_message() return 1 try: - sweep = DakotaDynamic(dakota_cfg, log_file_name, platform_filename, debug, ips_config_file, restart_file) + sweep = DakotaDynamic( + dakota_cfg, log_file_name, platform_filename, debug, ips_config_file, restart_file + ) sweep.run() except Exception: raise diff --git a/ipsframework/ips_es_spec.py b/ipsframework/ips_es_spec.py index eb36a3c9..f04293af 100644 --- a/ipsframework/ips_es_spec.py +++ b/ipsframework/ips_es_spec.py @@ -4,33 +4,33 @@ from .cca_es_spec import EventListener, PublisherEventService, SubscriberEventService -class myEventListener(EventListener): +class MyEventListener(EventListener): def __init__(self, callback_method): super().__init__() self.callback_method = callback_method - def processEvent(self, topicName, theEvent): - self.callback_method(topicName, theEvent) + def process_event(self, topic_name, the_event): + self.callback_method(topic_name, the_event) -class eventManager: +class EventManager: def __init__(self, obj_ref): self.obj_ref = obj_ref self.objcache = {} self.publisher = 'self.publisher' self.subscriber = 'self.subscriber' - def publish(self, topicName, eventName, eventBody): + def publish(self, topic_name, event_name, event_body): if self.publisher in self.objcache: pub = self.objcache[self.publisher] else: pub = PublisherEventService() self.objcache[self.publisher] = pub - topic = pub.getTopic(topicName) - topic.sendEvent(eventName, eventBody) + topic = pub.get_topic(topic_name) + topic.send_event(event_name, event_body) - def subscribe(self, topicName, callback): + def subscribe(self, topic_name, callback): if not callable(callback): callback_method = getattr(self.obj_ref, callback, None) else: @@ -47,29 +47,29 @@ def subscribe(self, topicName, callback): sub = SubscriberEventService() self.objcache[self.subscriber] = sub - if topicName in self.objcache: + if topic_name in self.objcache: # TODO: do we notify the client to do an unsubscribe before # re-subscribing to the same topic? the event service # currently throws an exception in this scenario... - scp = self.objcache[topicName] + scp = self.objcache[topic_name] else: - scp = sub.getSubscription(topicName) - self.objcache[topicName] = scp + scp = sub.get_subscription(topic_name) + self.objcache[topic_name] = scp - evl = myEventListener(callback_method) - scp.registerEventListener(topicName, evl) + evl = MyEventListener(callback_method) + scp.register_event_listener(topic_name, evl) - def unsubscribe(self, topicName): - if topicName in self.objcache: - self.objcache[topicName].unregisterEventListener(topicName) - del self.objcache[topicName] + def unsubscribe(self, topic_name): + if topic_name in self.objcache: + self.objcache[topic_name].unregister_event_listener(topic_name) + del self.objcache[topic_name] # else: # TODO: do we notify the client to do a subscribe first? # throw an exception? def process_events(self): if self.subscriber in self.objcache: - self.objcache[self.subscriber].processEvents() + self.objcache[self.subscriber].process_events() # else: # TODO: do we notify the client to do a subscribe before processing? # throw an exception? diff --git a/ipsframework/ipsExceptions.py b/ipsframework/ips_exceptions.py similarity index 82% rename from ipsframework/ipsExceptions.py rename to ipsframework/ips_exceptions.py index ef19de68..7aedebb5 100644 --- a/ipsframework/ipsExceptions.py +++ b/ipsframework/ips_exceptions.py @@ -3,7 +3,7 @@ # ------------------------------------------------------------------------------- -class BlockedMessageException(Exception): +class BlockedMessageError(Exception): """Exception Raised by the any manager when a blocking service invocation is made, and the invocation result is not readily available. @@ -19,21 +19,21 @@ def __str__(self): return 'message blocked because %s' % self.reason -class IncompleteCallException(Exception): +class IncompleteCallError(Exception): """Exception Raised by the taskManager when a nonblocking wait_call() method is invoked before the call has finished. """ - def __init__(self, callID): + def __init__(self, call_id): super().__init__() - self.callID = callID - self.args = (callID,) + self.call_id = call_id + self.args = (call_id,) def __str__(self): - return 'nonblocking wait_call() invoked before call %s finished' % self.callID + return 'nonblocking wait_call() invoked before call %s finished' % self.call_id -class InsufficientResourcesException(Exception): +class InsufficientResourcesError(Exception): """Exception Raised by the resource manager when not enough resources are available to satisfy an allocate() call """ @@ -60,7 +60,7 @@ def __str__(self): ) -class ResourceRequestMismatchException(Exception): +class ResourceRequestMismatchError(Exception): """Exception raised by the resource manager when it is possible to launch the requested number of processes, but not on the requested number of processes per node. @@ -79,12 +79,13 @@ def __init__(self, caller_id, tid, nproc, ppn, max_procs, max_ppn): def __str__(self): s = ( 'component %s requested %d processes with %d processes per node, while the number of processes requested ' - 'is less than the max (%d), the processes per node value is too low.' % (self.caller_id, self.nproc, self.ppn, self.max_procs) + 'is less than the max (%d), the processes per node value is too low.' + % (self.caller_id, self.nproc, self.ppn, self.max_procs) ) return s -class GPUResourceRequestMismatchException(Exception): +class GpuResourceRequestMismatchError(Exception): """Exception raised by the resource manager when it is possible to launch the requested number of GPUs per task """ @@ -99,16 +100,19 @@ def __init__(self, caller_id, tid, ppn, gpp, max_gpp): self.args = (caller_id, tid, ppn, gpp, max_gpp) def __str__(self): - s = 'component %s requested %d processes per node with %d GPUs per process, which is greater than the available %d GPUS_PER_NODE' % ( - self.caller_id, - self.ppn, - self.gpp, - self.max_gpp, + s = ( + 'component %s requested %d processes per node with %d GPUs per process, which is greater than the available %d GPUS_PER_NODE' + % ( + self.caller_id, + self.ppn, + self.gpp, + self.max_gpp, + ) ) return s -class ResourceRequestUnequalPartitioningException(Exception): +class ResourceRequestUnequalPartitioningError(Exception): """Exception raised by the resource manager when it is possible to launch the requested number of processes, but the requested number of processes and processes per node will result in unequal @@ -128,12 +132,13 @@ def __init__(self, caller_id, tid, nproc, ppn, max_procs, max_ppn): def __str__(self): s = ( 'component %s requested %d processes with %d processes per node, while the number of processes requested is less than the max (%d), ' - 'it will result in unequal partitioning of processes across nodes' % (self.caller_id, self.nproc, self.ppn, self.max_procs) + 'it will result in unequal partitioning of processes across nodes' + % (self.caller_id, self.nproc, self.ppn, self.max_procs) ) return s -class InvalidResourceSettingsException(Exception): +class InvalidResourceSettingsError(Exception): """ Exception raised by the resource helper to indicate inconsistent resource settings. """ @@ -147,14 +152,22 @@ def __init__(self, t, spn, cpn): def __str__(self): preamble = 'Invalid resource specification in platform configuration file: ' if self.type == 'spn > cpn': - return '%s socket per node count (%d) greater than core per node count (%d).' % (preamble, self.spn, self.cpn) + return '%s socket per node count (%d) greater than core per node count (%d).' % ( + preamble, + self.spn, + self.cpn, + ) elif self.type == 'spn not divisible by cpn': - return '%s socket per node count (%d) not divisible by core per node count (%d).' % (preamble, self.spn, self.cpn) + return '%s socket per node count (%d) not divisible by core per node count (%d).' % ( + preamble, + self.spn, + self.cpn, + ) else: return '%s unknown error' % (preamble) -class BadResourceRequestException(Exception): +class BadResourceRequestError(Exception): """Exception raised by the resource manager when a component requests a quantity of resources that can never be satisfied during a get_allocation() call diff --git a/ipsframework/ipsLogging.py b/ipsframework/ips_logging.py similarity index 85% rename from ipsframework/ipsLogging.py rename to ipsframework/ips_logging.py index f3b3237a..58dee510 100644 --- a/ipsframework/ipsLogging.py +++ b/ipsframework/ips_logging.py @@ -15,7 +15,7 @@ import time -class myLogRecordStreamHandler(socketserver.StreamRequestHandler): +class MyLogRecordStreamHandler(socketserver.StreamRequestHandler): def __init__(self, request, client_address, server, handler): self.handler = handler super().__init__(request, client_address, server) @@ -34,14 +34,14 @@ def handle(self): chunk = self.connection.recv(slen) while len(chunk) < slen: chunk = chunk + self.connection.recv(slen - len(chunk)) - obj = self.unPickle(chunk) + obj = self.un_pickle(chunk) record = logging.makeLogRecord(obj) - self.handleLogRecord(record) + self.handle_log_record(record) - def unPickle(self, data): + def un_pickle(self, data): return pickle.loads(data) - def handleLogRecord(self, record): + def handle_log_record(self, record): name = record.name logger = logging.getLogger(name) # Need to make sure we only have one handler, since the handler on the @@ -61,14 +61,14 @@ class LogRecordSocketReceiver(socketserver.ThreadingUnixStreamServer): allow_reuse_address = True - def __init__(self, log_pipe, handler=myLogRecordStreamHandler): + def __init__(self, log_pipe, handler=MyLogRecordStreamHandler): super().__init__(log_pipe, handler) def get_file_no(self): return self.socket.fileno() -class ipsLogger: +class IpsLogger: def __init__(self, dynamic_sim_queue=None): self.log_map = {} self.server_map = {} @@ -86,12 +86,16 @@ def add_sim_log(self, log_pipe_name, log_file=sys.stdout): try: os.makedirs(directory, exist_ok=True) except OSError as oserr: - print('Error creating directory %s : %s-%s' % (directory, oserr.errno, oserr.strerror), file=sys.stderr) + print( + 'Error creating directory %s : %s-%s' + % (directory, oserr.errno, oserr.strerror), + file=sys.stderr, + ) sys.exit(1) log_handler = logging.FileHandler(log_file, mode='w') log_handler.setFormatter(self.formatter) - partial_handler = functools.partial(myLogRecordStreamHandler, handler=log_handler) + partial_handler = functools.partial(MyLogRecordStreamHandler, handler=log_handler) recvr = LogRecordSocketReceiver(log_pipe_name, handler=partial_handler) fileno = recvr.get_file_no() self.log_map[fileno] = (recvr, log_handler, log_pipe_name) @@ -117,7 +121,9 @@ def __run__(self): pass else: tokens = msg.split() - if tokens[0] == 'CREATE_SIM': # Expecting Message: 'CREATE_SIM log_pipe_name log_file + if ( + tokens[0] == 'CREATE_SIM' + ): # Expecting Message: 'CREATE_SIM log_pipe_name log_file self.add_sim_log(tokens[1], tokens[2]) elif tokens[0] == 'END_SIM': # Expecting Message 'END_SIM log_pipe_name' log_pipe_name = tokens[1] diff --git a/ipsframework/ipsutil.py b/ipsframework/ipsutil.py index bc4ca5f3..90330f16 100644 --- a/ipsframework/ipsutil.py +++ b/ipsframework/ipsutil.py @@ -9,7 +9,7 @@ import shutil import sys import time -from typing import Iterable, Optional, Union +from collections.abc import Iterable try: import Pyro4 @@ -17,7 +17,7 @@ pass -def which(program, alt_paths: Optional[list[str]] = None): +def which(program, alt_paths: list[str] | None = None): def is_exe(fpath): return os.path.exists(fpath) and os.access(fpath, os.X_OK) @@ -39,7 +39,13 @@ def is_exe(fpath): return exe_file -def copyFiles(src_dir: str, src_file_list: Union[str, Iterable[str]], target_dir: str, prefix='', keep_old: bool = False): +def copy_files( + src_dir: str, + src_file_list: str | Iterable[str], + target_dir: str, + prefix='', + keep_old: bool = False, +): """ Copy files in *src_file_list* from *src_dir* to *target_dir* with an optional prefix. If *keep_old* is ``True``, existing files in @@ -51,7 +57,7 @@ def copyFiles(src_dir: str, src_file_list: Union[str, Iterable[str]], target_dir use_data_server = os.getenv('USE_DATA_SERVER', 'DATA_SERVER_NOT_USED') if use_data_server != 'DATA_SERVER_NOT_USED': data_server = Pyro4.Proxy('PYRONAME:DataServer') - data_server.copyFiles(src_dir, src_file_list, target_dir, prefix, keep_old) + data_server.copy_files(src_dir, src_file_list, target_dir, prefix, keep_old) return try: @@ -102,20 +108,20 @@ def copyFiles(src_dir: str, src_file_list: Union[str, Iterable[str]], target_dir raise -def getTimeString(timeArg: Optional[time.struct_time] = None): +def get_time_string(time_arg: time.struct_time | None = None): """ - Return a string representation of *timeArg*. *timeArg* is expected + Return a string representation of *time_arg*. *time_arg* is expected to be an appropriate object to be processed by :py:meth:`time.strftime`. - If *timeArg* is ``None``, current time is used. + If *time_arg* is ``None``, current time is used. """ - if timeArg is None: + if time_arg is None: arg = time.localtime() else: - arg = timeArg + arg = time_arg return time.strftime('%Y-%m-%d|%H:%M:%S%Z', arg) -def params_from_csv(infile: Union[str, os.PathLike]) -> dict[str, dict[str, list[str]]]: +def params_from_csv(infile: str | os.PathLike) -> dict[str, dict[str, list[str]]]: """ Read a CSV file and return a dictionary of parameters suitable for passing to services.run_ensemble() @@ -135,7 +141,7 @@ def params_from_csv(infile: Union[str, os.PathLike]) -> dict[str, dict[str, list The returned structure will look like this: - .. code-block:: python + .. code-block:: python variables = {'a_comp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], @@ -188,12 +194,12 @@ def group_ensemble_variables_into_instances(variables: dict[str, dict[str, list[ .. code-block:: python - [['prefix_0', [['a_sim_comp', {'A': 3, 'B': 2.34, 'C': 'bar'}], - ['another_sim_comp', {'D': 7, 'B': 0.775, 'F': 'xyzzy'}]]], - ['prefix_1', [['a_sim_comp', {'A': 2, 'B': 5.82, 'C': 'baz'}], - ['another_sim_comp', {'D': 5, 'B': 0.08, 'F': 'plud'}]]], - ['prefix_2', [['a_sim_comp', {'A': 4, 'B': 0.1, 'C': 'quux'}], - ['another_sim_comp', {'D': 9, 'B': 29.2, 'F': 'thud'}]]]] + [['prefix_0', [['ASimComp', {'A': 3, 'B': 2.34, 'C': 'bar'}], + ['AnotherSimComp', {'D': 7, 'B': 0.775, 'F': 'xyzzy'}]]], + ['prefix_1', [['ASimComp', {'A': 2, 'B': 5.82, 'C': 'baz'}], + ['AnotherSimComp', {'D': 5, 'B': 0.08, 'F': 'plud'}]]], + ['prefix_2', [['ASimComp', {'A': 4, 'B': 0.1, 'C': 'quux'}], + ['AnotherSimComp', {'D': 9, 'B': 29.2, 'F': 'thud'}]]]] prefix_n corresponds to a specific ensemble instance and will be used for a unique subdir name. That, in turn, references a @@ -206,19 +212,34 @@ def group_ensemble_variables_into_instances(variables: dict[str, dict[str, list[ # convert the list of variable values into corresponding dicts # mapping the variables to specific values. Sorta like a # column-wise to row-wise transposition. - transposed = {key: [dict(zip(inner.keys(), values)) for values in zip(*inner.values())] for key, inner in variables.items()} + transposed = { + key: [ + dict(zip(inner.keys(), values, strict=False)) + for values in zip(*inner.values(), strict=False) + ] + for key, inner in variables.items() + } # Build the final structure where each instance is named # {prefix}_n result = [ - (f'{name}{i}', [(sim_name, sim_data) for sim_name, sim_data_list in transposed.items() for sim_data in [sim_data_list[i]]]) + ( + f'{name}{i}', + [ + (sim_name, sim_data) + for sim_name, sim_data_list in transposed.items() + for sim_data in [sim_data_list[i]] + ], + ) for i in range(len(next(iter(transposed.values())))) ] return result -def ensemble_instances_to_csv(instances: list[tuple[str, list[tuple[str, dict[str, object]]]]], path: Union[str, os.PathLike]) -> None: +def ensemble_instances_to_csv( + instances: list[tuple[str, list[tuple[str, dict[str, object]]]]], path: str | os.PathLike +) -> None: """ Take in a structure of variables suitable for passing to services.run_ensemble(), and write a CSV file from it. @@ -252,12 +273,24 @@ def ensemble_instances_to_csv(instances: list[tuple[str, list[tuple[str, dict[st # header row writer.writerow( functools.reduce( - operator.iconcat, [[f'{instance[0]}:{component}' for component in list(instance[1].keys())] for instance in instances[0][1]], ['sim_name'] + operator.iconcat, + [ + [f'{instance[0]}:{component}' for component in list(instance[1].keys())] + for instance in instances[0][1] + ], + ['sim_name'], ) ) # data rows writer.writerows( - [functools.reduce(operator.iconcat, [list(component[1].values()) for component in instance[1]], [instance[0]]) for instance in instances] + [ + functools.reduce( + operator.iconcat, + [list(component[1].values()) for component in instance[1]], + [instance[0]], + ) + for instance in instances + ] ) diff --git a/ipsframework/messages.py b/ipsframework/messages.py index 314a3f4a..54d8c789 100644 --- a/ipsframework/messages.py +++ b/ipsframework/messages.py @@ -4,7 +4,7 @@ from typing import Literal -from ipsframework.componentRegistry import ComponentID +from ipsframework.component_registry import ComponentID class Message: @@ -27,7 +27,9 @@ def __init__(self, sender_id: ComponentID, receiver_id: ComponentID): def get_message_id(self): if self.message_id is None: delim = self.delimiter - self.message_id = delim.join([self.identifier, str(self.sender_id), str(self.receiver_id), str(self.counter)]) + self.message_id = delim.join( + [self.identifier, str(self.sender_id), str(self.receiver_id), str(self.counter)] + ) self.__class__.counter += 1 return self.message_id @@ -48,7 +50,15 @@ class ServiceRequestMessage(Message): delimiter = '|' identifier = 'REQUEST' - def __init__(self, sender_id: ComponentID, receiver_id: ComponentID, target_comp_id: ComponentID, target_method: str, *args, **keywords): + def __init__( + self, + sender_id: ComponentID, + receiver_id: ComponentID, + target_comp_id: ComponentID, + target_method: str, + *args, + **keywords, + ): super().__init__(sender_id, receiver_id) self.target_comp_id = target_comp_id self.target_method = target_method @@ -73,7 +83,14 @@ class ServiceResponseMessage(Message): delimiter = '|' identifier = 'RESPONSE' - def __init__(self, sender_id: ComponentID, receiver_id: ComponentID, request_msg_id: str, status: Literal[0, 1], *args): + def __init__( + self, + sender_id: ComponentID, + receiver_id: ComponentID, + request_msg_id: str, + status: Literal[0, 1], + *args, + ): super().__init__(sender_id, receiver_id) self.request_msg_id = request_msg_id self.status = status @@ -102,7 +119,15 @@ class MethodInvokeMessage(Message): delimiter = '|' identifier = 'INVOKE' - def __init__(self, sender_id: ComponentID, receiver_id: ComponentID, call_id: int, target_method: str, *args, **keywords): + def __init__( + self, + sender_id: ComponentID, + receiver_id: ComponentID, + call_id: int, + target_method: str, + *args, + **keywords, + ): super().__init__(sender_id, receiver_id) self.call_id = call_id self.target_method = target_method @@ -126,7 +151,14 @@ class MethodResultMessage(Message): delimiter = '|' identifier = 'RESULT' - def __init__(self, sender_id: ComponentID, receiver_id: ComponentID, call_id: int, status: Literal[0, 1], *args): + def __init__( + self, + sender_id: ComponentID, + receiver_id: ComponentID, + call_id: int, + status: Literal[0, 1], + *args, + ): super().__init__(sender_id, receiver_id) self.call_id = call_id self.args = args diff --git a/ipsframework/platformspec.py b/ipsframework/platformspec.py index c1d895a3..73e5eaa5 100644 --- a/ipsframework/platformspec.py +++ b/ipsframework/platformspec.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ The platform configuration file is used to specify the resources available to the framework for a given platform. @@ -9,27 +8,26 @@ # ------------------------------------------------------------------------------- import os import sys -from typing import Optional, Tuple from .messages import Message -def get_share_and_platform(platform_file_name: Optional[str], ipsPathName: str) -> Tuple[str, str]: +def get_share_and_platform(platform_file_name: str | None, ips_path_name: str) -> tuple[str, str]: if platform_file_name: return platform_file_name, '' else: - ipsPDir0 = os.path.dirname(ipsPathName) - ipsPDir1 = os.path.dirname(ipsPDir0) - ipsPDir2 = os.path.dirname(ipsPDir1) + ips_p_dir0 = os.path.dirname(ips_path_name) + ips_p_dir1 = os.path.dirname(ips_p_dir0) + ips_p_dir2 = os.path.dirname(ips_p_dir1) # This is if we've installed it pconf = os.path.join('share', 'platform.conf') - if os.path.exists(os.path.join(ipsPDir1, pconf)): - ipsShareDir = os.path.join(ipsPDir1, 'share') + if os.path.exists(os.path.join(ips_p_dir1, pconf)): + ips_share_dir = os.path.join(ips_p_dir1, 'share') # This is looking in the build directory. - elif os.path.exists(os.path.join(ipsPDir2, pconf)): - ipsShareDir = os.path.join(ipsPDir2, 'share') + elif os.path.exists(os.path.join(ips_p_dir2, pconf)): + ips_share_dir = os.path.join(ips_p_dir2, 'share') else: print('Need to specify a platform file', file=sys.stderr) sys.exit(Message.FAILURE) - platform_file_name = os.path.join(ipsShareDir, 'platform.conf') - return os.path.abspath(platform_file_name), ipsShareDir + platform_file_name = os.path.join(ips_share_dir, 'platform.conf') + return os.path.abspath(platform_file_name), ips_share_dir diff --git a/ipsframework/resourceHelper.py b/ipsframework/resource_helper.py similarity index 84% rename from ipsframework/resourceHelper.py rename to ipsframework/resource_helper.py index eed408fb..75843fb3 100644 --- a/ipsframework/resourceHelper.py +++ b/ipsframework/resource_helper.py @@ -12,7 +12,7 @@ import psutil -from .ipsExceptions import InvalidResourceSettingsException +from .ips_exceptions import InvalidResourceSettingsError def get_qstat_jobinfo(): @@ -66,7 +66,7 @@ def get_qstat_jobinfo(): num_procs = int(width[0].split('=')[1]) if len(mpp_npp) > 0: ppn = int(mpp_npp[0].split('=')[1]) - num_nodes = int(ceil(float(num_procs) / float(ppn))) + num_nodes = ceil(float(num_procs) / float(ppn)) return num_nodes, ppn, False, [] else: raise Exception('Error in call to qstat.') @@ -265,7 +265,7 @@ def get_pbs_info(): node_file = os.environ['PBS_NODEFILE'] # core_list is a misnomer, it is a list of (repeated) node names # where the node names are repeated for each process they can service - core_list_all = [line.strip() for line in open(node_file, 'r').readlines()] + core_list_all = [line.strip() for line in open(node_file, 'r')] core_list = core_list_all node_dict = {} for core in core_list: @@ -274,10 +274,10 @@ def get_pbs_info(): node_dict[core] += 1 except KeyError: node_dict[core] = 1 - listOfNodes = list(node_dict.items()) + list_of_nodes = list(node_dict.items()) max_p = max(node_dict.values()) mixed_nodes = max_p != min(node_dict.values()) - return len(listOfNodes), max_p, mixed_nodes, listOfNodes + return len(list_of_nodes), max_p, mixed_nodes, list_of_nodes except Exception: try: node_count = int(os.environ['PBS_NNODES']) @@ -290,7 +290,7 @@ def manual_detection(services): """ Use values listed in platform configuration file. """ - listOfNodes = [] + list_of_nodes = [] num_nodes = int(services.get_platform_parameter('NODES')) ppn = int(services.get_platform_parameter('PROCS_PER_NODE')) tot_procs = int(services.get_platform_parameter('TOTAL_PROCS')) @@ -305,86 +305,86 @@ def manual_detection(services): tot_procs = num_nodes * ppn for n in range(num_nodes): - listOfNodes.append(('dummynode%d' % n, ppn)) + list_of_nodes.append(('dummynode%d' % n, ppn)) if tot_procs < num_nodes * (ppn - 1): - n = listOfNodes[-1][0] - listOfNodes[-1] = (n, tot_procs % ppn) - return num_nodes, ppn, False, listOfNodes + n = list_of_nodes[-1][0] + list_of_nodes[-1] = (n, tot_procs % ppn) + return num_nodes, ppn, False, list_of_nodes -def getResourceList(services, host, partial_nodes=False): +def get_resource_list(services, host, partial_nodes=False): """ Using the host information, the resources are detected. Return list of (, ), cores per node, sockets per node, processes per node, and ``True`` if the node names are accurate, ``False`` otherwise. """ - listOfNodes = [] + list_of_nodes = [] # get the number of nodes for that machine num_nodes = 1 ppn = 1 spn = 1 cpn = 1 - accurateNodes = False + accurate_nodes = False mixed_nodes = False node_detect_str = services.get_platform_parameter('NODE_DETECTION', silent=True) if node_detect_str == 'checkjob': - num_nodes, ppn, mixed_nodes, listOfNodes = get_checkjob_info() + num_nodes, ppn, mixed_nodes, list_of_nodes = get_checkjob_info() print('=======================================================') - print(num_nodes, ppn, mixed_nodes, listOfNodes) - accurateNodes = False + print(num_nodes, ppn, mixed_nodes, list_of_nodes) + accurate_nodes = False elif node_detect_str == 'qstat': - num_nodes, ppn, mixed_nodes, listOfNodes = get_qstat_jobinfo() - accurateNodes = False + num_nodes, ppn, mixed_nodes, list_of_nodes = get_qstat_jobinfo() + accurate_nodes = False elif node_detect_str == 'qstat2': - num_nodes, ppn, mixed_nodes, listOfNodes = get_qstat_jobinfo2() - accurateNodes = True + num_nodes, ppn, mixed_nodes, list_of_nodes = get_qstat_jobinfo2() + accurate_nodes = True elif node_detect_str == 'pbs_env': - num_nodes, ppn, mixed_nodes, listOfNodes = get_pbs_info() + num_nodes, ppn, mixed_nodes, list_of_nodes = get_pbs_info() if ppn == 0: ppn = 1 - if not listOfNodes: + if not list_of_nodes: for n in range(num_nodes): - listOfNodes.append(('dummynode%d' % n, ppn)) + list_of_nodes.append(('dummynode%d' % n, ppn)) else: - accurateNodes = True + accurate_nodes = True elif node_detect_str == 'slurm_env': - num_nodes, ppn, mixed_nodes, listOfNodes = get_slurm_info() - accurateNodes = True + num_nodes, ppn, mixed_nodes, list_of_nodes = get_slurm_info() + accurate_nodes = True elif node_detect_str == 'manual': - num_nodes, ppn, mixed_nodes, listOfNodes = manual_detection(services) - accurateNodes = False + num_nodes, ppn, mixed_nodes, list_of_nodes = manual_detection(services) + accurate_nodes = False else: print( "WARNING: no node detection strategy specified in platform config file ('NODE_DETECTION'). " 'Valid options are: checkjob, qstat, pbs_env, slurm_env, manual. Trying all detection schemes.' ) try: - num_nodes, ppn, mixed_nodes, listOfNodes = get_checkjob_info() - accurateNodes = True + num_nodes, ppn, mixed_nodes, list_of_nodes = get_checkjob_info() + accurate_nodes = True except Exception: try: - num_nodes, ppn, mixed_nodes, listOfNodes = get_qstat_jobinfo() - accurateNodes = False + num_nodes, ppn, mixed_nodes, list_of_nodes = get_qstat_jobinfo() + accurate_nodes = False except Exception: try: - num_nodes, ppn, mixed_nodes, listOfNodes = get_pbs_info() + num_nodes, ppn, mixed_nodes, list_of_nodes = get_pbs_info() if ppn == 0: ppn = 1 - if not listOfNodes: + if not list_of_nodes: for n in range(num_nodes): - listOfNodes.append(('dummynode%d' % n, ppn)) + list_of_nodes.append(('dummynode%d' % n, ppn)) else: - accurateNodes = True + accurate_nodes = True except Exception: try: - num_nodes, ppn, mixed_nodes, listOfNodes = get_slurm_info() - accurateNodes = True + num_nodes, ppn, mixed_nodes, list_of_nodes = get_slurm_info() + accurate_nodes = True except Exception: try: - num_nodes, ppn, mixed_nodes, listOfNodes = manual_detection(services) - accurateNodes = False + num_nodes, ppn, mixed_nodes, list_of_nodes = manual_detection(services) + accurate_nodes = False except Exception: print('*** NO DETECTION MECHANISM WORKS ***') raise @@ -396,16 +396,16 @@ def getResourceList(services, host, partial_nodes=False): elif cpn < ppn: ppn = cpn if not mixed_nodes: - for i, node in enumerate(listOfNodes): + for i, node in enumerate(list_of_nodes): name = node[0] - listOfNodes[i] = (name, ppn) + list_of_nodes[i] = (name, ppn) if spn <= 0: spn = 1 elif spn > cpn: - raise InvalidResourceSettingsException('spn > cpn', spn, cpn) + raise InvalidResourceSettingsError('spn > cpn', spn, cpn) elif cpn % spn != 0: - raise InvalidResourceSettingsException('spn not divisible by cpn', spn, cpn) - return listOfNodes, cpn, spn, ppn, accurateNodes + raise InvalidResourceSettingsError('spn not divisible by cpn', spn, cpn) + return list_of_nodes, cpn, spn, ppn, accurate_nodes def get_platform_info(): @@ -419,9 +419,7 @@ def get_platform_info(): current running process, and available GPU devices if set; if the platform is supported it will also return CPU affinity """ - result = {'hostname': platform.node(), - 'cpu_count': psutil.cpu_count(), - 'pid': os.getpid()} + result = {'hostname': platform.node(), 'cpu_count': psutil.cpu_count(), 'pid': os.getpid()} if 'CUDA_VISIBLE_DEVICES' in os.environ: result['cuda_visible_devices'] = os.environ['CUDA_VISIBLE_DEVICES'] diff --git a/ipsframework/resourceManager.py b/ipsframework/resource_manager.py similarity index 77% rename from ipsframework/resourceManager.py rename to ipsframework/resource_manager.py index 0ffbc6c8..dddddeb4 100644 --- a/ipsframework/resourceManager.py +++ b/ipsframework/resource_manager.py @@ -6,20 +6,31 @@ import time from collections import namedtuple from math import ceil -from typing import Union - -from .ips_es_spec import eventManager -from .ipsExceptions import ( - BadResourceRequestException, - GPUResourceRequestMismatchException, - InsufficientResourcesException, - ResourceRequestMismatchException, - ResourceRequestUnequalPartitioningException, + +from .ips_es_spec import EventManager +from .ips_exceptions import ( + BadResourceRequestError, + GpuResourceRequestMismatchError, + InsufficientResourcesError, + ResourceRequestMismatchError, + ResourceRequestUnequalPartitioningError, ) from .node_structure import Node -from .resourceHelper import getResourceList - -Allocation = namedtuple('Allocation', ['partial_node', 'nodelist', 'corelist', 'ppn', 'max_ppn', 'cpp', 'accurateNodes', 'cores_allocated']) +from .resource_helper import get_resource_list + +Allocation = namedtuple( + 'Allocation', + [ + 'partial_node', + 'nodelist', + 'corelist', + 'ppn', + 'max_ppn', + 'cpp', + 'accurate_nodes', + 'cores_allocated', + ], +) class ResourceManager: @@ -43,7 +54,7 @@ def __init__(self, fwk): self.TM = None self.CM = None - self.accurateNodes = False + self.accurate_nodes = False self.node_alloc_mode = None self.host = None @@ -77,10 +88,10 @@ def __init__(self, fwk): # RM initialize - def initialize(self, dataMngr, taskMngr, configMngr, cmd_nodes=0, cmd_ppn=0): + def initialize(self, data_mngr, task_mngr, config_mngr, cmd_nodes=0, cmd_ppn=0): """ Initialize resource management structures, references to other - managers (*dataMngr*, *taskMngr*, *configMngr*). + managers (*data_mngr*, *task_mngr*, *config_mngr*). Resource information comes from the following in order of priority: @@ -89,12 +100,12 @@ def initialize(self, dataMngr, taskMngr, configMngr, cmd_nodes=0, cmd_ppn=0): * manual settings from platform config file The second two sources are obtained through - :py:meth:`resourceHelper.getResourceList`. + :py:meth:`resource_helper.get_resource_list`. """ - self.EM = eventManager(self) - self.DM = dataMngr - self.TM = taskMngr - self.CM = configMngr + self.EM = EventManager(self) + self.DM = data_mngr + self.TM = task_mngr + self.CM = config_mngr self.node_alloc_mode = self.CM.get_platform_parameter('NODE_ALLOCATION_MODE') rfile_name = os.path.join(self.CM.sim_map[self.CM.fwk_sim_name].sim_root, 'resource_usage') @@ -113,10 +124,10 @@ def initialize(self, dataMngr, taskMngr, configMngr, cmd_nodes=0, cmd_ppn=0): self.cores_per_node = int(cmd_ppn) self.ppn = int(cmd_ppn) self.sockets_per_node = 1 - self.accurateNodes = False - listOfNodes = [] + self.accurate_nodes = False + list_of_nodes = [] for i in range(cmd_nodes): - listOfNodes.append(('dummy_node%d' % i, cmd_ppn)) + list_of_nodes.append(('dummy_node%d' % i, cmd_ppn)) else: self.ppn = 0 # ------------------------------- @@ -125,12 +136,20 @@ def initialize(self, dataMngr, taskMngr, configMngr, cmd_nodes=0, cmd_ppn=0): self.host = self.CM.get_platform_parameter('HOST') self.fwk.debug('RM: Host = %s', self.host) try: - listOfNodes, self.cores_per_node, self.sockets_per_node, self.max_ppn, self.accurateNodes = getResourceList(self.CM, self.host) - self.fwk.warning('RM: listOfNodes = %s', str(listOfNodes)) + ( + list_of_nodes, + self.cores_per_node, + self.sockets_per_node, + self.max_ppn, + self.accurate_nodes, + ) = get_resource_list(self.CM, self.host) + self.fwk.warning('RM: list_of_nodes = %s', str(list_of_nodes)) self.fwk.warning('RM: max_ppn = %d ', int(self.max_ppn)) - if self.accurateNodes is True and not self.CM.get_platform_parameter('USE_ACCURATE_NODES'): - self.accurateNodes = False - self.fwk.warning('RM: User set accurateNodes to False') + if self.accurate_nodes is True and not self.CM.get_platform_parameter( + 'USE_ACCURATE_NODES' + ): + self.accurate_nodes = False + self.fwk.warning('RM: User set accurate_nodes to False') except Exception: print("can't get resource info") raise @@ -146,23 +165,28 @@ def initialize(self, dataMngr, taskMngr, configMngr, cmd_nodes=0, cmd_ppn=0): if user_ppn <= self.max_ppn: self.ppn = user_ppn - for i, (node, count) in enumerate(listOfNodes): + for i, (node, count) in enumerate(list_of_nodes): if count > self.ppn: - listOfNodes[i] = (node, self.ppn) + list_of_nodes[i] = (node, self.ppn) self.fwk.warning('Using user set procs per node: %d', user_ppn) else: - self.fwk.warning('Platform specified PROCS_PER_NODE = %d is greater than batch job specification = %d.' % (user_ppn, self.max_ppn)) + self.fwk.warning( + 'Platform specified PROCS_PER_NODE = %d is greater than batch job specification = %d.' + % (user_ppn, self.max_ppn) + ) self.fwk.warning('Will use batch job specification to launch tasks') self.ppn = self.max_ppn try: if user_ppn <= self.max_ppn: self.ppn = user_ppn - for i, (node, count) in enumerate(listOfNodes): + for i, (node, count) in enumerate(list_of_nodes): if count > self.ppn: - listOfNodes[i] = (node, self.ppn) + list_of_nodes[i] = (node, self.ppn) else: - self.fwk.warning('Platform specified PROCS_PER_NODE is greater than batch job specification.') + self.fwk.warning( + 'Platform specified PROCS_PER_NODE is greater than batch job specification.' + ) self.fwk.warning('Will use batch job specification to launch tasks') self.ppn = self.max_ppn except Exception: @@ -175,7 +199,10 @@ def initialize(self, dataMngr, taskMngr, configMngr, cmd_nodes=0, cmd_ppn=0): if (self.cores_per_node % self.sockets_per_node) == 0: self.cores_per_socket = self.cores_per_node // self.sockets_per_node else: - self.fwk.warning('cpn (%d) not divisible by spn(%d) - setting spn to 1' % (self.cores_per_node, self.sockets_per_node)) + self.fwk.warning( + 'cpn (%d) not divisible by spn(%d) - setting spn to 1' + % (self.cores_per_node, self.sockets_per_node) + ) self.sockets_per_node = 1 self.cores_per_socket = self.cores_per_node @@ -187,28 +214,37 @@ def initialize(self, dataMngr, taskMngr, configMngr, cmd_nodes=0, cmd_ppn=0): # ------------------------------- # populate nodes # ------------------------------- - self.fwk.warning('RM: %d nodes and %d processors per node' % (len(listOfNodes), self.ppn)) - self.total_cores = self.add_nodes(listOfNodes) + self.fwk.warning('RM: %d nodes and %d processors per node' % (len(list_of_nodes), self.ppn)) + self.total_cores = self.add_nodes(list_of_nodes) self.avail_cores = self.total_cores - self.begin_RM_report() + self.begin_rm_report() def process_service_request(self, msg): pass - def begin_RM_report(self): + def begin_rm_report(self): """ Print header information for resource usage reporting file. """ print('# host:', self.host, file=self.reporting_file) print('# total nodes:', self.num_nodes, file=self.reporting_file) print('# processors per node:', self.ppn, file=self.reporting_file) - print('using accurate nodes:', self.accurateNodes, file=self.reporting_file) - print('# time (in seconds since the | available | allocated | percent allocated | processes | percent used | notes ', file=self.reporting_file) - print('# resource manager started | | | | | |', file=self.reporting_file) - print('#-----------------------------------------------------------------------------------------------------------', file=self.reporting_file) - self.report_RM_status('initial state of resources') - - def report_RM_status(self, notes=''): + print('using accurate nodes:', self.accurate_nodes, file=self.reporting_file) + print( + '# time (in seconds since the | available | allocated | percent allocated | processes | percent used | notes ', + file=self.reporting_file, + ) + print( + '# resource manager started | | | | | |', + file=self.reporting_file, + ) + print( + '#-----------------------------------------------------------------------------------------------------------', + file=self.reporting_file, + ) + self.report_rm_status('initial state of resources') + + def report_rm_status(self, notes=''): """ Print current RM status to the reporting_file ("resource_usage") Entries consist of: @@ -221,16 +257,26 @@ def report_RM_status(self, notes=''): - % cores used by processes - notes (a description of the event that changed the resource usage) """ - print(' %27.5f |' % (time.time() - self.rm_start_of_time), end=' ', file=self.reporting_file) + print( + ' %27.5f |' % (time.time() - self.rm_start_of_time), end=' ', file=self.reporting_file + ) print(' %8d |' % self.avail_cores, end=' ', file=self.reporting_file) print(' %8d |' % self.alloc_cores, end=' ', file=self.reporting_file) - print(' %16.2f |' % (100 * (float(self.alloc_cores) / self.total_cores)), end=' ', file=self.reporting_file) + print( + ' %16.2f |' % (100 * (float(self.alloc_cores) / self.total_cores)), + end=' ', + file=self.reporting_file, + ) print(' %8d |' % self.processes, end=' ', file=self.reporting_file) - print(' %10.2f # ' % (100 * (float(self.processes) / self.total_cores)), end=' ', file=self.reporting_file) + print( + ' %10.2f # ' % (100 * (float(self.processes) / self.total_cores)), + end=' ', + file=self.reporting_file, + ) print(notes, file=self.reporting_file) self.reporting_file.flush() - def printRMState(self) -> None: + def print_rm_state(self) -> None: """ Print the node tree to ``stdout``. """ @@ -240,20 +286,20 @@ def printRMState(self) -> None: i.print_sockets() print('=====================') - def add_nodes(self, listOfNodes: list[tuple[str, int]]) -> int: + def add_nodes(self, list_of_nodes: list[tuple[str, int]]) -> int: """ Add node entries to ``self.nodes``. Typically used by :py:meth:`.initialize` to initialize ``self.nodes``. May be used to add nodes to a dynamic allocation in the future. - *listOfNodes* is a list of tuples (*node name*, *cores*). + *list_of_nodes* is a list of tuples (*node name*, *cores*). ``self.nodes`` is a dictionary where the keys are the *node names* and the values are :py:class:`node_structure.Node` structures. Return total number of cores. """ tot_cores = 0 - for n, p in listOfNodes: + for n, p in list_of_nodes: if n not in self.nodes: self.nodes.update({n: Node(n, self.sockets_per_node, self.cores_per_node, p)}) self.num_nodes += 1 @@ -266,7 +312,9 @@ def add_nodes(self, listOfNodes: list[tuple[str, int]]) -> int: # RM getAllocation # pylint: disable=inconsistent-return-statements - def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task_ppn=0, task_cpp=0, task_gpp=0): + def get_allocation( + self, comp_id, nproc, task_id, whole_nodes, whole_socks, task_ppn=0, task_cpp=0, task_gpp=0 + ): """ Traverse available nodes to return: @@ -276,7 +324,7 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task * *nodes*: list of node names * *ppn*: processes per node for launching the task * *max_ppn*: processes that can be launched - * *accurateNodes*: ``True`` if *nodes* uses the actual names of the nodes, ``False`` otherwise. + * *accurate_nodes*: ``True`` if *nodes* uses the actual names of the nodes, ``False`` otherwise. If *whole_nodes* is ``False``: @@ -286,7 +334,7 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task Core names are integers from 0 to n-1 where n is the number of cores on a node. * *ppn*: processes per node for launching the task * *max_ppn*: processes that can be launched - * *accurateNodes*: ``True`` if *nodes* uses the actual names of the nodes, ``False`` otherwise. + * *accurate_nodes*: ``True`` if *nodes* uses the actual names of the nodes, ``False`` otherwise. Arguments: @@ -316,7 +364,9 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task # check if partial node allocation is possible if self.node_alloc_mode == 'EXCLUSIVE': if not (whole_nodes and whole_socks): - self.fwk.warning('No partial node allocation available on this platform, using whole nodes instead.') + self.fwk.warning( + 'No partial node allocation available on this platform, using whole nodes instead.' + ) whole_nodes = True whole_socks = True @@ -349,17 +399,21 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task if not allocation_possible: if nodes == 'bad': c = ceil(float(nproc) / ppn) - raise BadResourceRequestException(comp_id, task_id, c, c - len(self.avail_nodes)) + raise BadResourceRequestError(comp_id, task_id, c, c - len(self.avail_nodes)) if nodes == 'mismatch': - raise ResourceRequestMismatchException(comp_id, task_id, nproc, ppn, self.total_cores, self.max_ppn) + raise ResourceRequestMismatchError( + comp_id, task_id, nproc, ppn, self.total_cores, self.max_ppn + ) if nodes == 'insufficient': c = ceil(float(nproc) / ppn) - raise InsufficientResourcesException(comp_id, task_id, c, c - len(self.avail_nodes)) + raise InsufficientResourcesError(comp_id, task_id, c, c - len(self.avail_nodes)) if nodes == 'unequal': - raise ResourceRequestUnequalPartitioningException(comp_id, task_id, nproc, ppn, self.total_cores, self.max_ppn) + raise ResourceRequestUnequalPartitioningError( + comp_id, task_id, nproc, ppn, self.total_cores, self.max_ppn + ) else: if not self.check_gpus(ppn, task_gpp): - raise GPUResourceRequestMismatchException(comp_id, task_id, ppn, task_gpp, self.gpn) + raise GpuResourceRequestMismatchError(comp_id, task_id, ppn, task_gpp, self.gpn) try: self.processes += nproc @@ -371,7 +425,9 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task # whole node allocation # ------------------------------- for n in nodes: - procs, cores = self.nodes[n].allocate(whole_nodes, whole_socks, task_id, comp_id, ppn) + procs, cores = self.nodes[n].allocate( + whole_nodes, whole_socks, task_id, comp_id, ppn + ) self.avail_nodes.remove(n) self.alloc_nodes.append(n) node_file_entries.append((n, cores)) @@ -387,7 +443,9 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task node = self.nodes[n] if node.avail_cores > 0: to_alloc = min([ppn, node.avail_cores, nproc - alloc_procs]) - procs, cores = node.allocate(whole_nodes, whole_socks, task_id, comp_id, to_alloc) + procs, cores = node.allocate( + whole_nodes, whole_socks, task_id, comp_id, to_alloc + ) cores_allocated += len(cores) alloc_procs = min([ppn, len(cores)]) node_file_entries.append((n, cores)) @@ -407,8 +465,12 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task node = self.nodes[n] if node.avail_cores > 0: to_alloc = min([ppn, node.avail_cores, nproc - cores_allocated]) - self.fwk.debug('allocate task_id %d node %s %d cores' % (task_id, n, to_alloc)) - procs, cores = node.allocate(whole_nodes, whole_socks, task_id, comp_id, to_alloc) + self.fwk.debug( + 'allocate task_id %d node %s %d cores' % (task_id, n, to_alloc) + ) + procs, cores = node.allocate( + whole_nodes, whole_socks, task_id, comp_id, to_alloc + ) cores_allocated += procs node_file_entries.append((n, cores)) if n not in self.alloc_nodes: @@ -438,7 +500,7 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task raise if whole_nodes: - self.report_RM_status('allocation for task %d using whole nodes' % task_id) + self.report_rm_status('allocation for task %d using whole nodes' % task_id) return Allocation( partial_node=not whole_nodes, nodelist=nodes, @@ -446,11 +508,11 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task ppn=ppn, max_ppn=self.max_ppn, cpp=cpp, - accurateNodes=self.accurateNodes, + accurate_nodes=self.accurate_nodes, cores_allocated=cores_allocated, ) else: - self.report_RM_status('allocation for task %d using partial nodes' % task_id) + self.report_rm_status('allocation for task %d using partial nodes' % task_id) return Allocation( partial_node=not whole_nodes, nodelist=nodes, @@ -458,7 +520,7 @@ def get_allocation(self, comp_id, nproc, task_id, whole_nodes, whole_socks, task ppn=ppn, max_ppn=self.max_ppn, cpp=None, - accurateNodes=self.accurateNodes, + accurate_nodes=self.accurate_nodes, cores_allocated=cores_allocated, ) @@ -543,7 +605,7 @@ def check_whole_sock_cap(self, nproc, ppn): else: return False, 'mismatch' - def check_core_cap(self, nproc: int, ppn: int) -> tuple[bool, Union[str, list[Node]]]: + def check_core_cap(self, nproc: int, ppn: int) -> tuple[bool, str | list[Node]]: """ Determine if it is currently possible to allocate *nproc* processes with a ppn of *ppn* without further restrictions.. Return ``True`` @@ -616,12 +678,12 @@ def release_allocation(self, task_id, status): self.alloc_cores -= num_cores self.processes -= nproc - self.report_RM_status('released nodes for task %d' % task_id) + self.report_rm_status('released nodes for task %d' % task_id) return True # RM SendEvent - def sendEvent(self, eventName, info): + def send_event(self, event_name, info): """ wrapper for constructing and publishing EM events """ @@ -629,7 +691,14 @@ def sendEvent(self, eventName, info): # send an event # ------------------------------- # populate event body - eventBody = {} - eventBody.update({'event name': eventName, 'topic': 'test', 'sender': 'RM', 'data': 'A resource event has occured'}) - eventBody.update(info) + event_body = {} + event_body.update( + { + 'event name': event_name, + 'topic': 'test', + 'sender': 'RM', + 'data': 'A resource event has occured', + } + ) + event_body.update(info) # send event on topic diff --git a/ipsframework/runspaceInitComponent.py b/ipsframework/runspace_init_component.py similarity index 71% rename from ipsframework/runspaceInitComponent.py rename to ipsframework/runspace_init_component.py index ebc710b0..0b68fc92 100644 --- a/ipsframework/runspaceInitComponent.py +++ b/ipsframework/runspace_init_component.py @@ -7,7 +7,7 @@ from ipsframework import Component, ipsutil -class runspaceInitComponent(Component): +class RunspaceInitComponent(Component): """ Framework component to manage runspace initialization, container file management, and file staging for simulation and analysis runs. @@ -48,8 +48,8 @@ def init(self, timestamp=0.0, **keywords): (head, _) = os.path.split(os.path.abspath(platform_file)) plat_file_loc = head - ipsutil.copyFiles(conf_file_loc, config_files, self.simRootDir) - ipsutil.copyFiles(plat_file_loc, platform_file, self.simRootDir) + ipsutil.copy_files(conf_file_loc, config_files, self.simRootDir) + ipsutil.copy_files(plat_file_loc, platform_file, self.simRootDir) def step(self, timestamp=0.0, **keywords): """ @@ -67,8 +67,15 @@ def step(self, timestamp=0.0, **keywords): # for each component_id in the list of components for comp_id in comp_list: # build the work directory name - comp_conf = registry.getEntry(comp_id).component_ref.config - full_comp_id = '_'.join([comp_conf['CLASS'], comp_conf['SUB_CLASS'], comp_conf['NAME'], str(comp_id.get_seq_num())]) + comp_conf = registry.get_entry(comp_id).component_ref.config + full_comp_id = '_'.join( + [ + comp_conf['CLASS'], + comp_conf['SUB_CLASS'], + comp_conf['NAME'], + str(comp_id.get_seq_num()), + ] + ) # compose the workdir name workdir = os.path.join(sim_roots[name], 'work', full_comp_id) @@ -77,12 +84,16 @@ def step(self, timestamp=0.0, **keywords): try: os.makedirs(workdir, exist_ok=True) except OSError as oserr: - self.services.exception('Error creating directory %s : %s', workdir, oserr.strerror) + self.services.exception( + 'Error creating directory %s : %s', workdir, oserr.strerror + ) raise # copy the input files into the working directory try: - ipsutil.copyFiles(os.path.abspath(comp_conf['INPUT_DIR']), comp_conf['INPUT_FILES'], workdir) + ipsutil.copy_files( + os.path.abspath(comp_conf['INPUT_DIR']), comp_conf['INPUT_FILES'], workdir + ) except Exception: print('Error copying input files for initialization', file=sys.stderr) raise @@ -90,6 +101,14 @@ def step(self, timestamp=0.0, **keywords): # copy the component's script to the simulation_setup directory if comp_conf['SCRIPT']: if os.path.isabs(comp_conf['SCRIPT']): - ipsutil.copyFiles(os.path.dirname(comp_conf['SCRIPT']), [os.path.basename(comp_conf['SCRIPT'])], simulation_setup) + ipsutil.copy_files( + os.path.dirname(comp_conf['SCRIPT']), + [os.path.basename(comp_conf['SCRIPT'])], + simulation_setup, + ) else: - ipsutil.copyFiles(comp_conf['BIN_DIR'], [os.path.basename(comp_conf['SCRIPT'])], simulation_setup) + ipsutil.copy_files( + comp_conf['BIN_DIR'], + [os.path.basename(comp_conf['SCRIPT'])], + simulation_setup, + ) diff --git a/ipsframework/services.py b/ipsframework/services.py index 0d805711..822afe4c 100644 --- a/ipsframework/services.py +++ b/ipsframework/services.py @@ -6,15 +6,11 @@ import functools import glob import hashlib -import json import logging import logging.handlers -from datetime import datetime import os -import platform import queue import shutil -import signal import socket import subprocess import sys @@ -23,31 +19,32 @@ import traceback import uuid import weakref +from collections.abc import Callable, Iterable from copy import deepcopy +from datetime import datetime from multiprocessing import Queue -from operator import iadd, itemgetter +from operator import iadd from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Iterable, NamedTuple, Optional, Union - -from rich import pretty -pretty.install() - -from rich.console import Console -console = Console() +from typing import TYPE_CHECKING, Any, NamedTuple import rich.traceback -from rich.traceback import Traceback -rich.traceback.install(show_locals=True) - from configobj import ConfigObj +from dask.distributed import get_worker from distributed import Client, Worker, WorkerPlugin +from rich import pretty +from rich.console import Console +from rich.traceback import Traceback from ipsframework import ipsutil, messages from ipsframework._internal.definitions import IPS_EVENT_TYPE from ipsframework.cca_es_spec import initialize_event_service -from ipsframework.componentRegistry import ComponentID -from ipsframework.ips_es_spec import eventManager -from ipsframework.taskManager import TaskInit +from ipsframework.component_registry import ComponentID +from ipsframework.ips_es_spec import EventManager +from ipsframework.task_manager import TaskInit + +pretty.install() +console = Console() +rich.traceback.install(show_locals=True) if TYPE_CHECKING: from ipsframework.component import Component @@ -64,11 +61,8 @@ class RunningTask(NamedTuple): args: list[str] -def launch(executable: Any, - task_name: str, - working_dir: Union[str, os.PathLike], - *args, **kwargs): - """ This is used by :meth:`TaskPool.submit_dask_tasks` as the +def launch(executable: Any, task_name: str, working_dir: str | os.PathLike, *args, **kwargs): + """This is used by :meth:`TaskPool.submit_dask_tasks` as the input to :meth:`dask.distributed.Client.submit`. Valid kwargs: @@ -92,9 +86,6 @@ def launch(executable: Any, :param working_dir: The working directory in which to run this task :returns: The task name and the return value from running the binary. """ - import logging - from dask.distributed import get_worker # pylint: disable=import-outside-toplevel - worker = get_worker() task_key = worker.get_current_task() @@ -102,10 +93,12 @@ def launch(executable: Any, # access the root logger for forward_logging() to work. log = logging.getLogger() - log.info(f'Launching task {task_name} with id {task_key!s} and ' - f'worker {worker.name!s} in {working_dir}') - print(f'Launching task {task_name} with id {task_key!s} and ' - f'worker {worker.name!s} in {working_dir}') + log.info( + f'Launching task {task_name} with id {task_key!s} and worker {worker.name!s} in {working_dir}' + ) + print( + f'Launching task {task_name} with id {task_key!s} and worker {worker.name!s} in {working_dir}' + ) start_time = time.time() working_dir_path = Path(working_dir) @@ -117,7 +110,7 @@ def launch(executable: Any, # Do we write the Popen stdout to sys.stdout or to a file? subprocess_stdout = subprocess.PIPE - close_stdout = False # is true if we need to later close the file + close_stdout = False # is true if we need to later close the file log_path = None try: log_filename = kwargs['logfile'] @@ -129,7 +122,7 @@ def launch(executable: Any, if not log_path.is_absolute(): log_path = working_dir_path / log_path subprocess_stdout = open(log_path, 'a') - close_stdout = True # Welp, gotta close it now + close_stdout = True # Welp, gotta close it now log.info(f'Task output log file: {log_path}') print(f'Task output log file: {log_path}') @@ -145,17 +138,17 @@ def launch(executable: Any, err_path = Path(err_filename) if not err_path.is_absolute(): err_path = working_dir_path / err_path - if log_path is not None and err_path.resolve(strict=False) == log_path.resolve(strict=False): + if log_path is not None and err_path.resolve(strict=False) == log_path.resolve( + strict=False + ): log.info(f'Task error log file matches output log file: {log_path}') print(f'Task error log file matches output log file: {log_path}') else: try: subprocess_stderr = open(err_path, 'a') except OSError: - log.info(f'Could not open errfile {err_path}, ' - f'using STDOUT for task errors') - print(f'Could not open errfile {err_path}, ' - f'using STDOUT for task errors') + log.info(f'Could not open errfile {err_path}, using STDOUT for task errors') + print(f'Could not open errfile {err_path}, using STDOUT for task errors') subprocess_stderr = subprocess.STDOUT else: close_stderr = True @@ -188,21 +181,21 @@ def launch(executable: Any, # in some HPC environments to ensure the output appears in the logs. if task_env is not None and task_env != {}: if 'PMIX_SERVER_URI41' in task_env: - log.debug(f"DVM environment variable PMIX_SERVER_URI41 " - f"set in task_env to " - f"{task_env['PMIX_SERVER_URI41']}") - print(f"DVM environment variable PMIX_SERVER_URI41 " - f"set in task_env to " - f"{task_env['PMIX_SERVER_URI41']}") + log.debug( + f'DVM environment variable PMIX_SERVER_URI41 set in task_env to {task_env["PMIX_SERVER_URI41"]}' + ) + print( + f'DVM environment variable PMIX_SERVER_URI41 set in task_env to {task_env["PMIX_SERVER_URI41"]}' + ) # print(f'DVM environment variable PMIX_SERVER_URI41 set in task_' # f'env to {task_env["PMIX_SERVER_URI41"]}', flush=True) if 'PMIX_SERVER_URI41' in os.environ: - log.debug(f"DVM environment variable PMIX_SERVER_URI41 set " - f"in os.environ to " - f"{os.environ['PMIX_SERVER_URI41']}") - print(f"DVM environment variable PMIX_SERVER_URI41 set " - f"in os.environ to " - f"{os.environ['PMIX_SERVER_URI41']}") + log.debug( + f'DVM environment variable PMIX_SERVER_URI41 set in os.environ to {os.environ["PMIX_SERVER_URI41"]}' + ) + print( + f'DVM environment variable PMIX_SERVER_URI41 set in os.environ to {os.environ["PMIX_SERVER_URI41"]}' + ) # print(f'DVM environment variable PMIX_SERVER_URI41 set in os.environ ' # f'to {os.environ["PMIX_SERVER_URI41"]}', flush=True) @@ -213,85 +206,85 @@ def launch(executable: Any, log.debug(f'Launching task {task_name} with command: {cmd}') print(f'Launching task {task_name} with command: {cmd}') - worker.log_event('ips', - { - 'eventType' : 'IPS_LAUNCH_DASK_TASK', - 'event_time': start_time, - 'state' : 'Running', - 'comment' : f'task_name = {task_name}, ' - f'Task key = {task_key!s}, ' - f'Target = {cmd}' - }) + worker.log_event( + 'ips', + { + 'event_type': 'IPS_LAUNCH_DASK_TASK', + 'event_time': start_time, + 'state': 'Running', + 'comment': f'task_name = {task_name}, Task key = {task_key!s}, Target = {cmd}', + }, + ) cmd_lst = cmd.split() process = None try: try: - process = subprocess.Popen(cmd_lst, - stdout=subprocess_stdout, - stderr=subprocess_stderr, - cwd=working_dir_path, - text=True, - preexec_fn=os.setsid, env=new_env) # noqa: PLW1509 (TODO: look into this to potentially avoid deadlocks) + process = subprocess.Popen( + cmd_lst, + stdout=subprocess_stdout, + stderr=subprocess_stderr, + cwd=working_dir_path, + text=True, + preexec_fn=os.setsid, # noqa: PLW1509 # TODO - FIX THIS, it is deprecated (https://github.com/python/cpython/issues/82616) (https://docs.astral.sh/ruff/rules/subprocess-popen-preexec-fn/) + env=new_env, + ) except Exception as e: - worker.log_event('ips', - { - 'eventType' : 'IPS_DASK_TASK_END', - 'event_time': time.time(), - 'state' : 'Failed', - 'comment' : f'task_name = {task_name} ' - f'Exception when calling ' - f'{executable!s}: {e!s}', - 'operation' : ' '.join(map(str, args)), - }) - log.error(f'Failed to launch task {task_name} with ' - f'command {cmd}: {e}') - print(f'Failed to launch task {task_name} with ' - f'command {cmd}: {e}') + worker.log_event( + 'ips', + { + 'event_type': 'IPS_DASK_TASK_END', + 'event_time': time.time(), + 'state': 'Failed', + 'comment': f'task_name = {task_name} Exception when calling {executable!s}: {e!s}', + 'operation': ' '.join(map(str, args)), + }, + ) + log.error(f'Failed to launch task {task_name} with command {cmd}: {e}') + print(f'Failed to launch task {task_name} with command {cmd}: {e}') raise try: ret_val = process.wait(timeout) finish_time = time.time() - worker.log_event('ips', - { - 'eventType' : 'IPS_DASK_TASK_END', - 'event_time' : finish_time, - 'state' : 'Succeeded', - 'comment' : f'task_name = ' - f'{task_name},' - f' elapsed time = ' - f'{finish_time - start_time:.2f}s', - 'start_time' : start_time, - 'elapsed_time': finish_time - start_time, - 'target' : executable, - 'operation' : ' '.join(map(str, args)), - }) + worker.log_event( + 'ips', + { + 'event_type': 'IPS_DASK_TASK_END', + 'event_time': finish_time, + 'state': 'Succeeded', + 'comment': f'task_name = {task_name}, elapsed time = {finish_time - start_time:.2f}s', + 'start_time': start_time, + 'elapsed_time': finish_time - start_time, + 'target': executable, + 'operation': ' '.join(map(str, args)), + }, + ) except subprocess.TimeoutExpired: - worker.log_event('ips', - { - 'eventType' : 'IPS_DASK_TASK_END', - 'event_time': time.time(), - 'state' : 'Timed out', - 'comment' : f'task_name = {task_name}, ' - f'timed-out after ' - f'{timeout}s'}) + worker.log_event( + 'ips', + { + 'event_type': 'IPS_DASK_TASK_END', + 'event_time': time.time(), + 'state': 'Timed out', + 'comment': f'task_name = {task_name}, timed-out after {timeout}s', + }, + ) process.kill() - log.error(f'Task {task_name} with command {cmd} timed out ' - f'after {timeout}s') - print(f'Task {task_name} with command {cmd} timed out ' - f'after {timeout}s') + log.error(f'Task {task_name} with command {cmd} timed out after {timeout}s') + print(f'Task {task_name} with command {cmd} timed out after {timeout}s') ret_val = -1 except Exception as e: - worker.log_event('ips', - { - 'eventType' : 'IPS_DASK_TASK_END', - 'event_time': time.time(), - 'state' : 'Failed', - 'comment' : f'task_name = {task_name} ' - f'Exception when calling ' - f'{executable!s}: {e!s}'}) + worker.log_event( + 'ips', + { + 'event_type': 'IPS_DASK_TASK_END', + 'event_time': time.time(), + 'state': 'Failed', + 'comment': f'task_name = {task_name} Exception when calling {executable!s}: {e!s}', + }, + ) log.error(f'Task {task_name} with command {cmd} failed with {e!s}') print(f'Task {task_name} with command {cmd} failed with {e!s}') finally: @@ -309,15 +302,15 @@ def launch(executable: Any, elif isinstance(executable, Callable): # binary not a string, but is a python callable, so we call it directly # with the given *args - worker.log_event('ips', - { - 'eventType' : 'IPS_LAUNCH_DASK_TASK', - 'event_time': time.time(), - 'state' : 'Running', - 'comment' : f'task_name = {task_name}, ' - f'Target = {executable.__name__}(' - f'{",".join(map(str, args))})', - }) + worker.log_event( + 'ips', + { + 'event_type': 'IPS_LAUNCH_DASK_TASK', + 'event_time': time.time(), + 'state': 'Running', + 'comment': f'task_name = {task_name}, Target = {executable.__name__}({",".join(map(str, args))})', + }, + ) try: original_dir = Path.cwd() @@ -327,38 +320,38 @@ def launch(executable: Any, finish_time = time.time() - worker.log_event('ips', - { - 'eventType' : 'IPS_DASK_TASK_END', - 'event_time' : finish_time, - 'state' : 'Succeeded', - 'comment' : f'task_name = {task_name}, ' - f'elapsed time = ' - f'{finish_time - start_time:.2f}s', - 'start_time' : start_time, - 'elapsed_time': finish_time - start_time, - 'target' : executable.__name__, - 'return_value': ret_val, - 'operation' : f'({",".join(map(str, args))})', - }) + worker.log_event( + 'ips', + { + 'event_type': 'IPS_DASK_TASK_END', + 'event_time': finish_time, + 'state': 'Succeeded', + 'comment': f'task_name = {task_name}, elapsed time = {finish_time - start_time:.2f}s', + 'start_time': start_time, + 'elapsed_time': finish_time - start_time, + 'target': executable.__name__, + 'return_value': ret_val, + 'operation': f'({",".join(map(str, args))})', + }, + ) except Exception as e: - worker.log_event('ips', - { - 'eventType' : 'IPS_DASK_TASK_END', - 'event_time': time.time(), - 'state' : 'Failed', - 'comment' : f'task_name = {task_name} ' - f'Exception when calling ' - f'{executable!s}: {e!s}'}) - log.error(f'Task {task_name} with callable {executable!s} failed ' - f'with {e!s}') - print(f'Task {task_name} with callable {executable!s} failed ' - f'with {e!s}') + worker.log_event( + 'ips', + { + 'event_type': 'IPS_DASK_TASK_END', + 'event_time': time.time(), + 'state': 'Failed', + 'comment': f'task_name = {task_name} Exception when calling {executable!s}: {e!s}', + }, + ) + log.error(f'Task {task_name} with callable {executable!s} failed with {e!s}') + print(f'Task {task_name} with callable {executable!s} failed with {e!s}') finally: os.chdir(str(original_dir)) else: - raise RuntimeError(f'Binary argument {executable!s} is not a string or ' - f'callable, cannot launch task {task_name}') + raise RuntimeError( + f'Binary argument {executable!s} is not a string or callable, cannot launch task {task_name}' + ) log.info(f'Task {task_name} finished with return value: {ret_val}') print(f'Task {task_name} finished with return value: {ret_val}') @@ -367,14 +360,15 @@ def launch(executable: Any, def launch_mapped_task( - executable: Any, - task_name: str, - working_dir: Union[str, os.PathLike], - task_args: Iterable[Any], - task_keywords: dict[str, Any], - cpus_per_proc: int, - worker_event_logfile: Optional[str]): - """ Adapt task-specific launch arguments for :meth:`Client.map`. + executable: Any, + task_name: str, + working_dir: str | os.PathLike, + task_args: Iterable[Any], + task_keywords: dict[str, Any], + cpus_per_proc: int, + worker_event_logfile: str | None, +): + """Adapt task-specific launch arguments for :meth:`Client.map`. This is a wrapper for `launch()` because we need to ensure `cpus_per_proc` and `worker_event_logfile` get stuffed into the expected `task_args` and @@ -431,7 +425,14 @@ class ServicesProxy: """ - def __init__(self, fwk, fwk_in_q: Queue, svc_response_q: Queue, sim_conf: dict[str, Any], log_pipe_name: str): + def __init__( + self, + fwk, + fwk_in_q: Queue, + svc_response_q: Queue, + sim_conf: dict[str, Any], + log_pipe_name: str, + ): self.pid = 0 self.fwk = fwk self.fwk_in_q = fwk_in_q @@ -471,7 +472,7 @@ def __init__(self, fwk, fwk_in_q: Queue, svc_response_q: Queue, sim_conf: dict[s self.shared_nodes = False self._portal_runid = -1 """This is the id we use on the portal to track this specific run. This will get set when receiving the IPS_START event from the portal. - + - Non-negative integer = successfully initialized - -1 = portal not yet contacted - -2 = portal initialization failed @@ -496,12 +497,19 @@ def __initialize__(self, component_ref): self.component_ref = weakref.proxy(component_ref) conf = self.component_ref.config - self.full_comp_id = '_'.join([conf['CLASS'], conf['SUB_CLASS'], conf['NAME'], str(self.component_ref.component_id.get_seq_num())]) + self.full_comp_id = '_'.join( + [ + conf['CLASS'], + conf['SUB_CLASS'], + conf['NAME'], + str(self.component_ref.component_id.get_seq_num()), + ] + ) self._ips_serialized_component_id = self.component_ref.component_id.get_serialization() # # Set up logging path to the IPS logging daemon # - socketHandler = logging.handlers.SocketHandler(self.log_pipe_name, None) + socket_handler = logging.handlers.SocketHandler(self.log_pipe_name, None) self.logger = logging.getLogger(self.full_comp_id) log_level = 'WARNING' try: @@ -516,8 +524,12 @@ def __initialize__(self, component_ref): except AttributeError: raise self.logger.setLevel(real_log_level) - self.logger.addHandler(socketHandler) - self.debug('__initialize__(): %s %s ', str(self.component_ref), str(self.component_ref.component_id)) + self.logger.addHandler(socket_handler) + self.debug( + '__initialize__(): %s %s ', + str(self.component_ref), + str(self.component_ref.component_id), + ) self.sim_name = self.component_ref.component_id.get_sim_name() # ------------------ # set shared_nodes @@ -550,24 +562,32 @@ def __initialize__(self, component_ref): if self.sim_conf['SIMULATION_MODE'] == 'RESTART': if self.sim_conf['RESTART_TIME'] == 'LATEST': chkpts = glob.glob(os.path.join(self.sim_conf['RESTART_ROOT'], 'restart', '*')) - base_dir = sorted(chkpts, key=lambda d: float(os.path.basename(d)))[-1] + base_dir = sorted(chkpts, key=lambda d: float(os.path.basename(d)))[-1] # noqa: FURB192 # TODO check this self.sim_conf['RESTART_TIME'] = os.path.basename(base_dir) def _init_event_service(self) -> None: """ Initialize connection to the central framework event service """ - self.debug('_init_event_service(): self.counter = %d - %s', self.counter, str(self.component_ref)) + self.debug( + '_init_event_service(): self.counter = %d - %s', self.counter, str(self.component_ref) + ) self.counter = self.counter + 1 initialize_event_service(self) - self.event_service = eventManager(self.component_ref) + self.event_service = EventManager(self.component_ref) - def _component_id_subscription_callback(self, topicName: str, theEvent) -> None: + def _component_id_subscription_callback(self, topic_name: str, the_event) -> None: """ All components subscribe to a topic based on their component_id. This function is the callback function for that subscription. """ - event_types = list(theEvent.getHeader().keys()) - self.debug('_component_id_subscription_callback(): component_id=%s topic=%s event_header=%s event_body=%s', self._ips_serialized_component_id, topicName, theEvent.getHeader(), theEvent.getBody()) + event_types = list(the_event.get_header().keys()) + self.debug( + '_component_id_subscription_callback(): component_id=%s topic=%s event_header=%s event_body=%s', + self._ips_serialized_component_id, + topic_name, + the_event.get_header(), + the_event.get_body(), + ) if '_IPS_PORTAL_UPLOAD_ENSEMBLE_PARAMS_SUCCESS' in event_types: self._ensemble_verification_check = True @@ -599,9 +619,7 @@ def _get_incoming_responses(self, block: bool = False) -> list[Any]: response = self.svc_response_q.get(block, timeout) response_list.append(response) except queue.Empty: - if not block: - finish = True - elif len(response_list) > 0: + if not block or len(response_list) > 0: finish = True return response_list @@ -659,7 +677,14 @@ def _invoke_service(self, component_id: ComponentID, method_name: str, *args, ** :return: message id """ self.debug('_invoke_service(): %s %s', method_name, str(args[0:])) - new_msg = messages.ServiceRequestMessage(self.component_ref.component_id, self.fwk.component_id, component_id, method_name, *args, **keywords) + new_msg = messages.ServiceRequestMessage( + self.component_ref.component_id, + self.fwk.component_id, + component_id, + method_name, + *args, + **keywords, + ) msg_id = new_msg.get_message_id() self.incomplete_calls[msg_id] = new_msg self.fwk_in_q.put(new_msg) @@ -696,36 +721,43 @@ def _get_service_response(self, msg_id, block=True): def _send_monitor_event( self, - eventType: IPS_EVENT_TYPE = '', + event_type: IPS_EVENT_TYPE = '', comment: str = '', ok: bool = True, state: str = 'Running', - event_time: Optional[float] = None, - elapsed_time: Optional[float] = None, - start_time: Optional[float] = None, - end_time: Optional[float] = None, - target: Optional[str] = None, - operation: Optional[str] = None, - procs_requested: Optional[int] = None, - cores_allocated: Optional[int] = None, + event_time: float | None = None, + elapsed_time: float | None = None, + start_time: float | None = None, + end_time: float | None = None, + target: str | None = None, + operation: str | None = None, + procs_requested: int | None = None, + cores_allocated: int | None = None, call_id: int = 0, ) -> None: """ Construct and send an event populated with the component's - information, *eventType*, *comment*, *ok*, *state*, and a wall time + information, *event_type*, *comment*, *ok*, *state*, and a wall time stamp, to the portal bridge to pass on to the web portal. """ portal_data = {} - portal_data['code'] = f'{self.component_ref.CLASS}_{self.component_ref.SUB_CLASS}_{self.component_ref.NAME}' - portal_data['eventtype'] = eventType + portal_data['code'] = ( + f'{self.component_ref.CLASS}_{self.component_ref.SUB_CLASS}_{self.component_ref.NAME}' + ) + portal_data['eventtype'] = event_type portal_data['ok'] = ok if event_time is None: event_time = time.time() portal_data['walltime'] = '%.2f' % (event_time - self.component_ref.start_time) - portal_data['time'] = ipsutil.getTimeString(time.localtime(event_time)) + portal_data['time'] = ipsutil.get_time_string(time.localtime(event_time)) trace = {} # Zipkin json format - if start_time is not None and (elapsed_time is not None or end_time is not None) and target is not None and operation is not None: + if ( + start_time is not None + and (elapsed_time is not None or end_time is not None) + and target is not None + and operation is not None + ): trace['timestamp'] = int(start_time * 1e6) # convert to microsecond if elapsed_time is not None: trace['duration'] = int(elapsed_time * 1e6) @@ -733,7 +765,9 @@ def _send_monitor_event( trace['duration'] = int((end_time - start_time) * 1e6) # convert to microsecond trace['localEndpoint'] = {'serviceName': target} trace['name'] = operation - formatted_args = ['%.3f' % (x) if isinstance(x, float) else str(x) for x in self.component_ref.args] + formatted_args = [ + '%.3f' % (x) if isinstance(x, float) else str(x) for x in self.component_ref.args + ] trace['id'] = hashlib.md5(f'{target}:{operation}:{call_id}'.encode()).hexdigest()[:16] trace['parentId'] = hashlib.md5( f'{self.component_ref.component_id}:{self.component_ref.method_name}({" ,".join(formatted_args)}):{self.component_ref.call_id}'.encode() @@ -765,7 +799,7 @@ def get_port(self, port_name: str) -> ComponentID: :type port_name: str :return: Return a reference to the component implementing port *port_name*. - :rtype: :class:`ipsframework.componentRegistry.ComponentID` + :rtype: :class:`ipsframework.component_registry.ComponentID` """ msg_id = self._invoke_service(self.fwk.component_id, 'get_port', port_name) response = self._get_service_response(msg_id, True) @@ -782,12 +816,14 @@ def cleanup(self): except Exception: pass - def call_nonblocking(self, component_id: ComponentID, method_name: str, *args, **keywords) -> int: + def call_nonblocking( + self, component_id: ComponentID, method_name: str, *args, **keywords + ) -> int: r"""Invoke method *method_name* on component *component_id* with optional arguments *\*args*. Will not wait until finished. :param component_id: Component ID of requested component - :type component_id: :class:`~ipsframework.componentRegistry.ComponentID` + :type component_id: :class:`~ipsframework.component_registry.ComponentID` :param method_name: component method to call, e.g. ``init`` or ``step`` :type method_name: str @@ -799,7 +835,10 @@ def call_nonblocking(self, component_id: ComponentID, method_name: str, *args, * formatted_args = ['%.3f' % (x) if isinstance(x, float) else str(x) for x in args] if keywords: formatted_args += ['%s=' % k + str(v) for (k, v) in keywords.items()] - self._send_monitor_event('IPS_CALL_BEGIN', 'Target = ' + target + ':' + method_name + '(' + ' ,'.join(formatted_args) + ')') + self._send_monitor_event( + 'IPS_CALL_BEGIN', + 'Target = ' + target + ':' + method_name + '(' + ' ,'.join(formatted_args) + ')', + ) msg_id = self._invoke_service(component_id, 'init_call', method_name, *args, **keywords) call_id = self._get_service_response(msg_id, True) self.call_targets[call_id] = (target, method_name, args, time.time()) @@ -811,7 +850,7 @@ def call(self, component_id: ComponentID, method_name: str, *args, **keywords): finished. Return result from invoking the method. :param component_id: Component ID of requested component - :type component_id: :class:`~ipsframework.componentRegistry.ComponentID` + :type component_id: :class:`~ipsframework.component_registry.ComponentID` :param method_name: component method to call, e.g. ``init`` or ``step`` :type method_name: str @@ -826,7 +865,7 @@ def call(self, component_id: ComponentID, method_name: str, *args, **keywords): def wait_call(self, call_id: int, block: bool = True): """If *block* is ``True``, return when the call has completed with the return code from the call. If *block* is ``False``, raise - :exc:`~ipsframework.ipsExceptions.IncompleteCallException` if + :exc:`~ipsframework.ips_exceptions.IncompleteCallError` if the call has not completed, and the return value is it has. :param call_id: call ID @@ -876,7 +915,7 @@ def wait_call_list(self, call_id_list: list[int], block=True): """Check the status of each of the call in *call_id_list*. If *block* is ``True``, return when *all* calls are finished. If *block* is ``False``, raise - :exc:`~ipsframework.ipsExceptions.IncompleteCallException` if + :exc:`~ipsframework.ips_exceptions.IncompleteCallError` if *any* of the calls have not completed, otherwise return. The return value is a dictionary of *call_ids* and return values. @@ -937,10 +976,10 @@ def launch_task(self, nproc: int, working_dir: str, binary: str, *args, **keywor Return *task_id* if successful. May raise exceptions related to opening the logfile, being unable to obtain enough resources to launch - the task (:exc:`~ipsframework.ipsExceptions.InsufficientResourcesException`), bad + the task (:exc:`~ipsframework.ips_exceptions.InsufficientResourcesError`), bad task launch request - (:exc:`~ipsframework.ipsExceptions.ResourceRequestMismatchException`, - :exc:`~ipsframework.ipsExceptions.BadResourceRequestException`) or problems + (:exc:`~ipsframework.ips_exceptions.ResourceRequestMismatchError`, + :exc:`~ipsframework.ips_exceptions.BadResourceRequestError`) or problems executing the command. These exceptions may be used to retry launching the task as appropriate. @@ -960,8 +999,13 @@ def launch_task(self, nproc: int, working_dir: str, binary: str, *args, **keywor """ if not isinstance(binary, str): - self.error('Error in launch_task: task binary of wrong type, expected str but found %s', type(binary).__name__) - raise ValueError(f'task binary of wrong type, expected str but found {type(binary).__name__}') + self.error( + 'Error in launch_task: task binary of wrong type, expected str but found %s', + type(binary).__name__, + ) + raise ValueError( + f'task binary of wrong type, expected str but found {type(binary).__name__}' + ) args = tuple(str(a) for a in args) tokens = binary.split() @@ -1020,7 +1064,9 @@ def launch_task(self, nproc: int, working_dir: str, binary: str, *args, **keywor launch_cmd_extra_args, ), ) - (task_id, command, env_update, cores_allocated) = self._get_service_response(msg_id, block=True) + (task_id, command, env_update, cores_allocated) = self._get_service_response( + msg_id, block=True + ) self.debug(f'init_task(): task_id = {task_id}') self.debug(f'command = {command}') self.debug(f'env_update = {env_update}') @@ -1029,7 +1075,18 @@ def launch_task(self, nproc: int, working_dir: str, binary: str, *args, **keywor self.error(f'Error setting up task for command "{command}": {e}') raise - task_id = self._launch_task(nproc, working_dir, task_id, command, cores_allocated, env_update, tag, keywords, binary, args) + task_id = self._launch_task( + nproc, + working_dir, + task_id, + command, + cores_allocated, + env_update, + tag, + keywords, + binary, + args, + ) self.debug(f'Returned task_id = {task_id} for launching "{command}"') @@ -1051,7 +1108,17 @@ def launch_task(self, nproc: int, working_dir: str, binary: str, *args, **keywor return task_id def _launch_task( - self, nproc: int, working_dir: str, task_id, command, cores_allocated, env_update, tag, keywords, binary: str, args: Union[list[str], tuple[str, ...]] + self, + nproc: int, + working_dir: str, + task_id, + command, + cores_allocated, + env_update, + tag, + keywords, + binary: str, + args: list[str] | tuple[str, ...], ): log_filename = keywords.get('logfile') timeout = keywords.get('timeout', 1.0e9) @@ -1084,16 +1151,22 @@ def _launch_task( if env_update: new_env = os.environ.copy() new_env.update(env_update) - process = subprocess.Popen(cmd_lst, stdout=task_stdout, stderr=task_stderr, cwd=working_dir, env=new_env) + process = subprocess.Popen( + cmd_lst, stdout=task_stdout, stderr=task_stderr, cwd=working_dir, env=new_env + ) else: - process = subprocess.Popen(cmd_lst, stdout=task_stdout, stderr=task_stderr, cwd=working_dir) + process = subprocess.Popen( + cmd_lst, stdout=task_stdout, stderr=task_stderr, cwd=working_dir + ) except Exception: self.exception('Error executing command : %s', command) raise # FIXME: process Monitoring Command : ps --no-headers -o pid,state pid1 pid2 pid3 ... - self.task_map[task_id] = RunningTask(process, time.time(), timeout, nproc, cores_allocated, command, binary, args) + self.task_map[task_id] = RunningTask( + process, time.time(), timeout, nproc, cores_allocated, command, binary, args + ) return task_id # process.pid def launch_task_pool(self, task_pool_name: str, launch_interval: float = 0.0) -> dict[str, Any]: @@ -1122,7 +1195,9 @@ def launch_task_pool(self, task_pool_name: str, launch_interval: float = 0.0) -> task_name, type(task.binary).__name__, ) - raise ValueError(f'task {task_name} binary of wrong type, expected str but found {type(task.binary).__name__}') + raise ValueError( + f'task {task_name} binary of wrong type, expected str but found {type(task.binary).__name__}' + ) task_ppn = task.keywords.get('task_ppn', self.ppn) wnodes = task.keywords.get('whole_nodes', not self.shared_nodes) wsocks = task.keywords.get('whole_sockets', not self.shared_nodes) @@ -1131,7 +1206,18 @@ def launch_task_pool(self, task_pool_name: str, launch_interval: float = 0.0) -> omp = task.keywords.get('omp', False) launch_cmd_extra_args = task.keywords.get('launch_cmd_extra_args') submit_dict[task_name] = TaskInit( - task.nproc, task.binary, task.working_dir, task_ppn, task_cpp, task_gpp, False, omp, wnodes, wsocks, task.args, launch_cmd_extra_args + task.nproc, + task.binary, + task.working_dir, + task_ppn, + task_cpp, + task_gpp, + False, + omp, + wnodes, + wsocks, + task.args, + launch_cmd_extra_args, ) try: @@ -1150,7 +1236,16 @@ def launch_task_pool(self, task_pool_name: str, launch_interval: float = 0.0) -> tag = task.keywords.get('tag', 'None') active_tasks[task_name] = self._launch_task( - task.nproc, task.working_dir, task_id, command, cores_allocated, env_update, tag, task.keywords, task.binary, task.args + task.nproc, + task.working_dir, + task_id, + command, + cores_allocated, + env_update, + tag, + task.keywords, + task.binary, + task.args, ) if env_update: @@ -1197,7 +1292,9 @@ def kill_task(self, task_id: int) -> None: del self.task_map[task_id] try: - msg_id = self._invoke_service(self.fwk.component_id, 'finish_task', task_id, task_retval) + msg_id = self._invoke_service( + self.fwk.component_id, 'finish_task', task_id, task_retval + ) self._get_service_response(msg_id, block=True) except Exception: self.exception('Error finalizing task %s', task_id) @@ -1214,7 +1311,7 @@ def kill_all_tasks(self) -> None: except Exception: raise - def wait_task_nonblocking(self, task_id: int) -> Union[int, None]: + def wait_task_nonblocking(self, task_id: int) -> int | None: """Check the status of task *task_id*. If it has finished, the return value is populated with the actual value, otherwise ``None`` is returned. A *KeyError* exception may be raised if @@ -1236,7 +1333,11 @@ def wait_task_nonblocking(self, task_id: int) -> Union[int, None]: if task_retval is None: if task.start_time + task.timeout < time.time(): self.kill_task(task_id) - self._send_monitor_event('IPS_TASK_END', 'source_func = wait_task_nonblocking, task_id = %s TIMEOUT elapsed time = %.2f S' % (str(task_id), time.time() - task.start_time)) + self._send_monitor_event( + 'IPS_TASK_END', + 'source_func = wait_task_nonblocking, task_id = %s TIMEOUT elapsed time = %.2f S' + % (str(task_id), time.time() - task.start_time), + ) return -1 else: return None @@ -1282,9 +1383,15 @@ def wait_task(self, task_id: int, timeout: int = -1, delay: int = 1) -> int: if task_retval is None: task.process.kill() task_retval = task.process.wait() - event_comment = 'source_func = wait_task, task_id = %s TIMEOUT elapsed time = %.2f S' % (str(task_id), finish_time - task.start_time) + event_comment = ( + 'source_func = wait_task, task_id = %s TIMEOUT elapsed time = %.2f S' + % (str(task_id), finish_time - task.start_time) + ) else: - event_comment = 'source_func = wait_task, task_id = %s elapsed time = %.2f S' % (str(task_id), finish_time - task.start_time) + event_comment = 'source_func = wait_task, task_id = %s elapsed time = %.2f S' % ( + str(task_id), + finish_time - task.start_time, + ) self._send_monitor_event( 'IPS_TASK_END', @@ -1301,7 +1408,9 @@ def wait_task(self, task_id: int, timeout: int = -1, delay: int = 1) -> int: del self.task_map[task_id] try: - msg_id = self._invoke_service(self.fwk.component_id, 'finish_task', task_id, task_retval) + msg_id = self._invoke_service( + self.fwk.component_id, 'finish_task', task_id, task_retval + ) self._get_service_response(msg_id, block=True) except Exception: self.exception('Error finalizing task %s', task_id) @@ -1398,11 +1507,11 @@ def get_config_param(self, param: str, silent: bool = False, log: bool = True) - return None return val - def set_config_param(self, param: str, value: Any, target_sim_name: Optional[str] = None) -> Any: + def set_config_param(self, param: str, value: Any, target_sim_name: str | None = None) -> Any: """Set configuration parameter *param* to *value*. Raise exceptions if the parameter cannot be changed or if there are problems setting the value. This tell the framework to call - :meth:`ipsframework.configurationManager.ConfigurationManager.set_config_parameter` + :meth:`ipsframework.configuration_manager.ConfigurationManager.set_config_parameter` to change the parameter. :param param: The parameter requested from simulation config @@ -1420,7 +1529,9 @@ def set_config_param(self, param: str, value: Any, target_sim_name: Optional[str if param in self.sim_conf: raise Exception('Cannot dynamically alter simulation configuration parameter ' + param) try: - msg_id = self._invoke_service(self.fwk.component_id, 'set_config_parameter', param, value, sim_name) + msg_id = self._invoke_service( + self.fwk.component_id, 'set_config_parameter', param, value, sim_name + ) retval = self._get_service_response(msg_id, block=True) except Exception: self.exception('Error setting value of configuration parameter %s', param) @@ -1447,7 +1558,9 @@ def safe(nums): for entry in ['FINISH', 'START', 'NSTEP']: if not safe(time_conf[entry]): self.error('Invalid TIME_LOOP value of %s = %s', entry, time_conf[entry]) - raise ValueError('Invalid TIME_LOOP value of %s = %s' % (entry, time_conf[entry])) + raise ValueError( + 'Invalid TIME_LOOP value of %s = %s' % (entry, time_conf[entry]) + ) finish = float(eval(time_conf['FINISH'])) start = float(eval(time_conf['START'])) nstep = int(eval(time_conf['NSTEP'])) @@ -1459,13 +1572,13 @@ def safe(nums): self.time_loop = tlist return tlist - def checkpoint_components(self, comp_id_list, time_stamp, Force=False, Protect=False): + def checkpoint_components(self, comp_id_list, time_stamp, force=False, protect=False): """ Selectively checkpoint components in *comp_id_list* based on the - configuration section *CHECKPOINT*. If *Force* is ``True``, the + configuration section *CHECKPOINT*. If *force* is ``True``, the checkpoint will be taken even if the conditions for taking the - checkpoint are not met. If *Protect* is ``True``, then the data from - the checkpoint is protected from clean up. *Force* and *Protect* are + checkpoint are not met. If *protect* is ``True``, then the data from + the checkpoint is protected from clean up. *force* and *protect* are optional and default to ``False``. The *CHECKPOINT_MODE* option controls determines if the components @@ -1475,7 +1588,7 @@ def checkpoint_components(self, comp_id_list, time_stamp, Force=False, Protect=F ALL: Checkpint every time the call is made (equivalent to always setting - Force =True) + force =True) WALLTIME_REGULAR: checkpoints are saved upon invocation of the service call ``checkpoint_components()``, when a time interval greater than, or @@ -1520,38 +1633,46 @@ def checkpoint_components(self, comp_id_list, time_stamp, Force=False, Protect=F based on their creation time. Possible values of NUM_CHECKPOINT are: * NUM_CHECKPOINT = n, with n > 0 --> Keep the most recent n checkpoints - * NUM_CHECKPOINT = 0 --> No checkpoints are made/kept (except when *Force* = ``True``) + * NUM_CHECKPOINT = 0 --> No checkpoints are made/kept (except when *force* = ``True``) * NUM_CHECKPOINT < 0 --> Keep ALL checkpoints Checkpoints are saved in the directory ``${SIM_ROOT}/restart`` """ elapsed_time = self._get_elapsed_time() - if Force: - return self._dispatch_checkpoint(time_stamp, comp_id_list, Protect) + if force: + return self._dispatch_checkpoint(time_stamp, comp_id_list, protect) try: chkpt_conf = self.sim_conf['CHECKPOINT'] mode = chkpt_conf['MODE'] num_chkpt = int(chkpt_conf['NUM_CHECKPOINT']) except KeyError: - self.error('Missing CHECKPOINT config section, or one of the required parameters: MODE, NUM_CHECKPOINT') + self.error( + 'Missing CHECKPOINT config section, or one of the required parameters: MODE, NUM_CHECKPOINT' + ) self.exception('Error accessing CHECKPOINT section in config file') raise if num_chkpt == 0: return None - if mode not in ['ALL', 'WALLTIME_REGULAR', 'WALLTIME_EXPLICIT', 'PHYSTIME_REGULAR', 'PHYSTIME_EXPLICIT']: + if mode not in [ + 'ALL', + 'WALLTIME_REGULAR', + 'WALLTIME_EXPLICIT', + 'PHYSTIME_REGULAR', + 'PHYSTIME_EXPLICIT', + ]: self.error('Invalid MODE = %s in checkpoint configuration', mode) raise Exception('Invalid MODE = %s in checkpoint configuration' % (mode)) if mode == 'ALL': - return self._dispatch_checkpoint(time_stamp, comp_id_list, Protect) + return self._dispatch_checkpoint(time_stamp, comp_id_list, protect) if mode == 'WALLTIME_REGULAR': interval = float(chkpt_conf['WALLTIME_INTERVAL']) if self.cur_time - self.last_ckpt_walltime >= interval: - return self._dispatch_checkpoint(time_stamp, comp_id_list, Protect) + return self._dispatch_checkpoint(time_stamp, comp_id_list, protect) else: return None elif mode == 'WALLTIME_EXPLICIT': @@ -1563,7 +1684,7 @@ def checkpoint_components(self, comp_id_list, time_stamp, Force=False, Protect=F wt_values = [float(t) for t in wt_values] for t in wt_values: if elapsed_time >= t > self.last_ckpt_walltime - self.start_time: - return self._dispatch_checkpoint(time_stamp, comp_id_list, Protect) + return self._dispatch_checkpoint(time_stamp, comp_id_list, protect) return None elif mode == 'PHYSTIME_REGULAR': pt_interval = float(chkpt_conf['PHYSTIME_INTERVAL']) @@ -1572,7 +1693,7 @@ def checkpoint_components(self, comp_id_list, time_stamp, Force=False, Protect=F if self.last_ckpt_phystime is None: self.last_ckpt_phystime = pt_start if pt_current - self.last_ckpt_phystime >= pt_interval: - return self._dispatch_checkpoint(time_stamp, comp_id_list, Protect) + return self._dispatch_checkpoint(time_stamp, comp_id_list, protect) else: return None elif mode == 'PHYSTIME_EXPLICIT': @@ -1584,21 +1705,25 @@ def checkpoint_components(self, comp_id_list, time_stamp, Force=False, Protect=F pt_current = float(time_stamp) for pt in pt_values: if pt_current >= pt > self.last_ckpt_phystime: - return self._dispatch_checkpoint(time_stamp, comp_id_list, Protect) + return self._dispatch_checkpoint(time_stamp, comp_id_list, protect) return None return None - def _dispatch_checkpoint(self, time_stamp, comp_id_list, Protect): + def _dispatch_checkpoint(self, time_stamp, comp_id_list, protect): """ Invoke *checkpoint* method on each component in *comp_id_list* labeled - with time *time_stamp*. If *Protect* is ``True``, or this checkpoint + with time *time_stamp*. If *protect* is ``True``, or this checkpoint is designated as a protected checkpoint by the simulation configuration parameters, steps are taken to ensure it remains in the restart directory. Unprotected checkpoints are purged as necessary. """ self.last_ckpt_walltime = self.cur_time self.last_ckpt_phystime = float(time_stamp) - self.debug('Checkpointing components after %.3f sec with physics time = %.3f', self.last_ckpt_walltime - self.start_time, self.last_ckpt_phystime) + self.debug( + 'Checkpointing components after %.3f sec with physics time = %.3f', + self.last_ckpt_walltime - self.start_time, + self.last_ckpt_phystime, + ) self._send_monitor_event('IPS_CHECKPOINT_START', 'Components = ' + str(comp_id_list)) call_id_list = [] for comp_id in comp_id_list: @@ -1617,21 +1742,29 @@ def _dispatch_checkpoint(self, time_stamp, comp_id_list, Protect): return ret_dict base_dir = os.path.join(sim_root, 'restart') - timeStamp_str = '%0.3f' % (float(time_stamp)) - self.new_chkpts.append(timeStamp_str) + time_stamp_str = '%0.3f' % (float(time_stamp)) + self.new_chkpts.append(time_stamp_str) try: protect_freq = chkpt_conf['PROTECT_FREQUENCY'] except KeyError: pass else: - if Protect or (self.chkpt_counter % int(protect_freq) == 0): - self.protected_chkpts.append(timeStamp_str) + if protect or (self.chkpt_counter % int(protect_freq) == 0): + self.protected_chkpts.append(time_stamp_str) if os.path.isdir(base_dir): - all_chkpts = [os.path.basename(f) for f in glob.glob(os.path.join(base_dir, '*')) if os.path.isdir(f)] + all_chkpts = [ + os.path.basename(f) + for f in glob.glob(os.path.join(base_dir, '*')) + if os.path.isdir(f) + ] prior_runs_chkpts_dirs = [chkpt for chkpt in all_chkpts if chkpt not in self.new_chkpts] purge_candidates = sorted(prior_runs_chkpts_dirs, key=float) - purge_candidates += [chkpt for chkpt in self.new_chkpts if (chkpt in all_chkpts and chkpt not in self.protected_chkpts)] + purge_candidates += [ + chkpt + for chkpt in self.new_chkpts + if (chkpt in all_chkpts and chkpt not in self.protected_chkpts) + ] while len(purge_candidates) > num_chkpt: obsolete_chkpt = purge_candidates.pop(0) chkpt_dir = os.path.join(base_dir, obsolete_chkpt) @@ -1663,14 +1796,14 @@ def get_working_dir(self) -> str: return self.workdir # DM stageInput - def stage_input_files(self, input_file_list: Union[str, Iterable[str]]) -> None: + def stage_input_files(self, input_file_list: str | Iterable[str]) -> None: """ Copy component input files to the component working directory (as obtained via a call to :py:meth:`ServicesProxy.get_working_dir`). Input files are assumed to be originally located in the directory variable *INPUT_DIR* in the component configuration section. - File are copied using :func:`ipsframework.ipsutil.copyFiles`. + File are copied using :func:`ipsframework.ipsutil.copy_files`. :param input_file_list: input files can space separated string or iterable of strings :type input_file_list: str or Iterable of str @@ -1678,8 +1811,8 @@ def stage_input_files(self, input_file_list: Union[str, Iterable[str]]) -> None: start_time = time.time() workdir = self.get_working_dir() old_conf = self.component_ref.config - inputDir = old_conf['INPUT_DIR'] - ipsutil.copyFiles(inputDir, input_file_list, workdir) + input_dir = old_conf['INPUT_DIR'] + ipsutil.copy_files(input_dir, input_file_list, workdir) # Copy input files into a central place in the output tree simroot = self.sim_conf['SIM_ROOT'] @@ -1690,9 +1823,13 @@ def stage_input_files(self, input_file_list: Union[str, Iterable[str]]) -> None: targetdir = os.path.join(simroot, 'simulation_setup', self.full_comp_id) try: - ipsutil.copyFiles(inputDir, input_file_list, targetdir, outprefix) + ipsutil.copy_files(input_dir, input_file_list, targetdir, outprefix) except Exception as e: - self._send_monitor_event('IPS_STAGE_INPUTS', 'Files = ' + str(input_file_list) + ' Exception raised : ' + str(e), ok=False) + self._send_monitor_event( + 'IPS_STAGE_INPUTS', + 'Files = ' + str(input_file_list) + ' Exception raised : ' + str(e), + ok=False, + ) self.exception('Error in stage_input_files') raise e for _, old_conf, _, _ in self.sub_flows.values(): @@ -1704,15 +1841,20 @@ def stage_input_files(self, input_file_list: Union[str, Iterable[str]]) -> None: input_target_dir = os.path.join(os.getcwd(), c) os.makedirs(input_target_dir, exist_ok=True) try: - ipsutil.copyFiles(input_dir, input_files, input_target_dir) + ipsutil.copy_files(input_dir, input_files, input_target_dir) except Exception as e: - self._send_monitor_event('IPS_STAGE_INPUTS', 'Files = ' + str(input_files) + ' Exception raised : ' + str(e), ok=False) + self._send_monitor_event( + 'IPS_STAGE_INPUTS', + 'Files = ' + str(input_files) + ' Exception raised : ' + str(e), + ok=False, + ) self.exception('Error in stage_input_files') raise e elapsed_time = time.time() - start_time self._send_monitor_event( - eventType='IPS_STAGE_INPUTS', - comment='Elapsed time = %.3f Path = %s Files = %s' % (elapsed_time, os.path.abspath(inputDir), str(input_file_list)), + event_type='IPS_STAGE_INPUTS', + comment='Elapsed time = %.3f Path = %s Files = %s' + % (elapsed_time, os.path.abspath(input_dir), str(input_file_list)), start_time=start_time, elapsed_time=elapsed_time, target='stage_input_files', @@ -1741,13 +1883,26 @@ def stage_subflow_output_files(self, subflow_name: str = 'ALL') -> dict[str, lis for sim_name, (sub_conf_new, _, _, driver_comp) in subflow_dict.items(): driver = sub_conf_new[sub_conf_new['PORTS']['DRIVER']['IMPLEMENTATION']] output_dir = os.path.join( - sub_conf_new['SIM_ROOT'], 'work', '_'.join([driver['CLASS'], driver['SUB_CLASS'], driver['NAME'], str(driver_comp.get_seq_num())]) + sub_conf_new['SIM_ROOT'], + 'work', + '_'.join( + [ + driver['CLASS'], + driver['SUB_CLASS'], + driver['NAME'], + str(driver_comp.get_seq_num()), + ] + ), ) output_files = driver['OUTPUT_FILES'] try: - ipsutil.copyFiles(output_dir, output_files, self.get_working_dir(), keep_old=False) + ipsutil.copy_files(output_dir, output_files, self.get_working_dir(), keep_old=False) except Exception as e: - self._send_monitor_event('IPS_STAGE_SUBFLOW_OUTPUTS', 'Files = ' + str(output_files) + ' Exception raised : ' + str(e), ok=False) + self._send_monitor_event( + 'IPS_STAGE_SUBFLOW_OUTPUTS', + 'Files = ' + str(output_files) + ' Exception raised : ' + str(e), + ok=False, + ) self.exception('Error in stage_subflow_output_files() for subflow %s' % sim_name) raise else: @@ -1757,18 +1912,24 @@ def stage_subflow_output_files(self, subflow_name: str = 'ALL') -> dict[str, lis return_dict[sim_name] = output_files return return_dict - def stage_output_files(self, timeStamp: float, file_list: Union[str, list[str]], keep_old_files: bool = True, save_plasma_state: bool = True) -> None: + def stage_output_files( + self, + time_stamp: float, + file_list: str | list[str], + keep_old_files: bool = True, + save_plasma_state: bool = True, + ) -> None: """ Copy associated component output files (from the working directory) to the component simulation results directory. Output files are prefixed with the configuration parameter *OUTPUT_PREFIX*. The simulation results directory has the format:: - ${SIM_ROOT}/simulation_results//components/$CLASS_${SUB_CLASS}_$NAME_${SEQ_NUM} + ${SIM_ROOT}/simulation_results//components/$CLASS_${SUB_CLASS}_$NAME_${SEQ_NUM} Additionally, plasma state files are archived for debugging purposes:: - ${SIM_ROOT}/history/plasma_state/_$CLASS_${SUB_CLASS}_$NAME_ + ${SIM_ROOT}/history/plasma_state/_$CLASS_${SUB_CLASS}_$NAME_ Copying errors are not fatal (exception raised). """ @@ -1782,14 +1943,20 @@ def stage_output_files(self, timeStamp: float, file_list: Union[str, list[str]], outprefix = '' out_root = 'simulation_results' - output_dir = os.path.join(sim_root, out_root, str(timeStamp), 'components', self.full_comp_id) + output_dir = os.path.join( + sim_root, out_root, str(time_stamp), 'components', self.full_comp_id + ) if isinstance(file_list, str): file_list = file_list.split() all_files = functools.reduce(iadd, [glob.glob(f) for f in file_list], []) try: - ipsutil.copyFiles(workdir, all_files, output_dir, outprefix, keep_old=keep_old_files) + ipsutil.copy_files(workdir, all_files, output_dir, outprefix, keep_old=keep_old_files) except Exception as e: - self._send_monitor_event('IPS_STAGE_OUTPUTS', 'Files = ' + str(file_list) + ' Exception raised : ' + str(e), ok=False) + self._send_monitor_event( + 'IPS_STAGE_OUTPUTS', + 'Files = ' + str(file_list) + ' Exception raised : ' + str(e), + ok=False, + ) self.exception('Error in stage_output_files()') raise @@ -1802,7 +1969,11 @@ def stage_output_files(self, timeStamp: float, file_list: Union[str, list[str]], try: os.makedirs(plasma_dir, exist_ok=True) except OSError as e: - self._send_monitor_event('IPS_STAGE_OUTPUTS', 'Files = ' + str(file_list) + ' Exception raised : ' + e.strerror, ok=False) + self._send_monitor_event( + 'IPS_STAGE_OUTPUTS', + 'Files = ' + str(file_list) + ' Exception raised : ' + e.strerror, + ok=False, + ) self.exception('Error creating directory %s : %d-%s', plasma_dir, e.errno, e.strerror) raise @@ -1822,24 +1993,32 @@ def stage_output_files(self, timeStamp: float, file_list: Union[str, list[str]], continue tokens = f.split('.') if len(tokens) == 1: - newName = '_'.join([outprefix + f, self.full_comp_id, str(timeStamp)]) + new_name = '_'.join([outprefix + f, self.full_comp_id, str(time_stamp)]) else: name = '.'.join(tokens[:-1]) ext = tokens[-1] - newName = '_'.join([outprefix + name, self.full_comp_id, str(timeStamp)]) + '.' + ext - target_name = os.path.join(plasma_dir, newName) + new_name = ( + '_'.join([outprefix + name, self.full_comp_id, str(time_stamp)]) + '.' + ext + ) + target_name = os.path.join(plasma_dir, new_name) if os.path.isfile(target_name): for i in range(1000): - newName = target_name + '.' + str(i) - if os.path.isfile(newName): + new_name = target_name + '.' + str(i) + if os.path.isfile(new_name): continue - target_name = newName + target_name = new_name break try: shutil.copy(f, target_name) - except (IOError, os.error) as why: - self.exception('Error copying file: %s from %s to %s - %s', f, workdir, target_name, str(why)) - self._send_monitor_event('IPS_STAGE_OUTPUTS', 'Files = ' + str(file_list) + ' Exception raised : ' + str(why), ok=False) + except OSError as why: + self.exception( + 'Error copying file: %s from %s to %s - %s', f, workdir, target_name, str(why) + ) + self._send_monitor_event( + 'IPS_STAGE_OUTPUTS', + 'Files = ' + str(file_list) + ' Exception raised : ' + str(why), + ok=False, + ) raise # Store symlinks to component output files in a single top-level directory @@ -1857,12 +2036,12 @@ def stage_output_files(self, timeStamp: float, file_list: Union[str, list[str]], real_file = os.path.join(output_dir, outprefix + f) tokens = f.rsplit('.', 1) if len(tokens) == 1: - newName = '_'.join([f, str(timeStamp)]) + new_name = '_'.join([f, str(time_stamp)]) else: name = tokens[0] ext = tokens[1] - newName = '_'.join([name, str(timeStamp)]) + '.' + ext - sym_link = os.path.join(symlink_dir, newName) + new_name = '_'.join([name, str(time_stamp)]) + '.' + ext + sym_link = os.path.join(symlink_dir, new_name) if os.path.isfile(sym_link): os.remove(sym_link) # We need to use relative path for the symlinks @@ -1888,7 +2067,7 @@ def stage_output_files(self, timeStamp: float, file_list: Union[str, list[str]], operation=str(file_list), ) - def save_restart_files(self, timeStamp: float, file_list: Union[str, list[str]]) -> None: + def save_restart_files(self, time_stamp: float, file_list: str | list[str]) -> None: """ Copy files needed for component restart to the restart directory:: @@ -1907,26 +2086,34 @@ def save_restart_files(self, timeStamp: float, file_list: Union[str, list[str]]) return conf = self.component_ref.config base_dir = os.path.join(sim_root, 'restart') - timeStamp_str = '%0.3f' % (float(timeStamp)) - self.new_chkpts.append(timeStamp_str) + time_stamp_str = '%0.3f' % (float(time_stamp)) + self.new_chkpts.append(time_stamp_str) - targetdir = os.path.join(base_dir, timeStamp_str, '_'.join([conf['CLASS'], conf['SUB_CLASS'], conf['NAME']])) + targetdir = os.path.join( + base_dir, time_stamp_str, '_'.join([conf['CLASS'], conf['SUB_CLASS'], conf['NAME']]) + ) self.debug('Checkpointing: Copying %s to dir %s', str(file_list), targetdir) try: - ipsutil.copyFiles(workdir, file_list, targetdir) + ipsutil.copy_files(workdir, file_list, targetdir) except Exception as e: - self._send_monitor_event('IPS_STAGE_RESTART', 'Files = ' + str(file_list) + ' Exception raised : ' + str(e), ok=False) + self._send_monitor_event( + 'IPS_STAGE_RESTART', + 'Files = ' + str(file_list) + ' Exception raised : ' + str(e), + ok=False, + ) self.exception('Error in stage_restart_files()') raise self._send_monitor_event('IPS_SAVE_RESTART', 'Files = ' + str(file_list)) - def get_restart_files(self, restart_root: str, timeStamp: float, file_list: Union[str, list[str]]) -> None: + def get_restart_files( + self, restart_root: str, time_stamp: float, file_list: str | list[str] + ) -> None: """ Copy files needed for component restart from the restart directory:: - /restart//components/$CLASS_${SUB_CLASS}_$NAME_${SEQ_NUM} + /restart//components/$CLASS_${SUB_CLASS}_$NAME_${SEQ_NUM} to the component's work directory. @@ -1935,19 +2122,25 @@ def get_restart_files(self, restart_root: str, timeStamp: float, file_list: Unio work_dir = self.get_working_dir() conf = self.component_ref.config - base_dir = os.path.join(restart_root, 'restart', '%.3f' % (float(timeStamp))) - source_dir = os.path.join(base_dir, '_'.join([conf['CLASS'], conf['SUB_CLASS'], conf['NAME']])) + base_dir = os.path.join(restart_root, 'restart', '%.3f' % (float(time_stamp))) + source_dir = os.path.join( + base_dir, '_'.join([conf['CLASS'], conf['SUB_CLASS'], conf['NAME']]) + ) try: - ipsutil.copyFiles(source_dir, file_list, work_dir) + ipsutil.copy_files(source_dir, file_list, work_dir) except Exception as e: - self._send_monitor_event('IPS_GET_RESTART', 'Files = ' + str(file_list) + ' Exception raised : ' + str(e), ok=False) + self._send_monitor_event( + 'IPS_GET_RESTART', + 'Files = ' + str(file_list) + ' Exception raised : ' + str(e), + ok=False, + ) self.exception('Error in get_restart_files()') raise self._send_monitor_event('IPS_GET_RESTART', 'Files = ' + str(file_list)) - def stage_state(self, state_files: Optional[list[str]] = None) -> None: + def stage_state(self, state_files: list[str] | None = None) -> None: """ Copy current state to work directory. """ @@ -1965,7 +2158,9 @@ def stage_state(self, state_files: Optional[list[str]] = None) -> None: workdir = self.get_working_dir() try: - msg_id = self._invoke_service(self.fwk.component_id, 'stage_state', files, state_dir, workdir) + msg_id = self._invoke_service( + self.fwk.component_id, 'stage_state', files, state_dir, workdir + ) self._get_service_response(msg_id, block=True) except Exception as e: self._send_monitor_event('IPS_STAGE_STATE', ' Exception raised : ' + str(e), ok=False) @@ -1981,7 +2176,7 @@ def stage_state(self, state_files: Optional[list[str]] = None) -> None: operation=str(files), ) - def update_state(self, state_files: Optional[list[str]] = None) -> None: + def update_state(self, state_files: list[str] | None = None) -> None: """ Copy local (updated) state to global state. If no state files are specified, component configuration specification is used. @@ -2001,7 +2196,9 @@ def update_state(self, state_files: Optional[list[str]] = None) -> None: state_dir = self.get_config_param('STATE_WORK_DIR') workdir = self.get_working_dir() try: - msg_id = self._invoke_service(self.fwk.component_id, 'update_state', files, workdir, state_dir) + msg_id = self._invoke_service( + self.fwk.component_id, 'update_state', files, workdir, state_dir + ) self._get_service_response(msg_id, block=True) except Exception as e: print('Error updating state files', str(e), file=sys.stderr) @@ -2018,7 +2215,12 @@ def update_state(self, state_files: Optional[list[str]] = None) -> None: operation=str(files), ) - def merge_current_state(self, partial_state_file: str, logfile: Optional[str] = None, merge_binary: Optional[str] = None) -> None: + def merge_current_state( + self, + partial_state_file: str, + logfile: str | None = None, + merge_binary: str | None = None, + ) -> None: """ Merge partial plasma state with global state. Partial plasma state contains only the values that the component contributes to the @@ -2041,20 +2243,38 @@ def merge_current_state(self, partial_state_file: str, logfile: Optional[str] = self.error('Missing executable %s in PATH', bin_name) raise FileNotFoundError('Missing executable file %s in PATH' % bin_name) try: - msg_id = self._invoke_service(self.fwk.component_id, 'merge_current_plasma_state', update_file, source_plasma_file, logfile, full_path_binary) + msg_id = self._invoke_service( + self.fwk.component_id, + 'merge_current_plasma_state', + update_file, + source_plasma_file, + logfile, + full_path_binary, + ) ret_val = self._get_service_response(msg_id, block=True) except Exception as e: print('Error merging state files', str(e), file=sys.stderr) - self._send_monitor_event('IPS_MERGE_PLASMA_STATE', ' Exception raised : ' + str(e), ok=False) + self._send_monitor_event( + 'IPS_MERGE_PLASMA_STATE', ' Exception raised : ' + str(e), ok=False + ) self.exception('Error merging plasma state file ' + partial_state_file) raise if ret_val == 0: self._send_monitor_event('IPS_MERGE_PLASMA_STATE', 'Success') return else: - self._send_monitor_event('IPS_MERGE_PLASMA_STATE', ' Error in call to update_state() : ', ok=False) - self.error('Error merging update %s into current plasma state file %s', partial_state_file, current_plasma_state) - raise Exception('Error merging update %s into current plasma state file %s' % (partial_state_file, current_plasma_state)) + self._send_monitor_event( + 'IPS_MERGE_PLASMA_STATE', ' Error in call to update_state() : ', ok=False + ) + self.error( + 'Error merging update %s into current plasma state file %s', + partial_state_file, + current_plasma_state, + ) + raise Exception( + 'Error merging update %s into current plasma state file %s' + % (partial_state_file, current_plasma_state) + ) def update_time_stamp(self, new_time_stamp=-1) -> None: """ @@ -2072,20 +2292,20 @@ def update_time_stamp(self, new_time_stamp=-1) -> None: self.publish('_IPS_MONITOR', 'PORTALBRIDGE_UPDATE_TIMESTAMP', event_data) self._send_monitor_event('IPS_UPDATE_TIME_STAMP', 'Timestamp = ' + str(new_time_stamp)) - def setMonitorURL(self, url: str = '') -> None: + def set_monitor_url(self, url: str = '') -> None: """ Send event to portal setting the URL where the monitor component will put data. """ self.monitor_url = url - self._send_monitor_event(eventType='IPS_SET_MONITOR_URL', comment='SUCCESS') + self._send_monitor_event(event_type='IPS_SET_MONITOR_URL', comment='SUCCESS') def _should_use_portal(self) -> bool: """Return True if we want to use the portal, False if not""" use_portal_config = self.get_config_param('USE_PORTAL', silent=True) if isinstance(use_portal_config, str): - return use_portal_config.strip().lower() == 'true' + return use_portal_config.strip().lower() == 'true' elif use_portal_config is None: return False else: @@ -2117,14 +2337,18 @@ def _establish_portal_runid(self) -> None: # next, check to see if the portal URL was even initialized, fall back if not if not self.get_config_param('PORTAL_URL', silent=True): - self.warning('_get_jupyter_runid: PORTAL_URL was not defined, disabling Jupyter workflow') + self.warning( + '_get_jupyter_runid: PORTAL_URL was not defined, disabling Jupyter workflow' + ) self._portal_runid = -2 self._portal_runid_event.set() return - + # next, check to see if the user remembered to define an API key (adding data requires a runid) if not self.get_config_param('_IPS_PORTAL_API_KEY', silent=True): - self.warning('_get_jupyter_runid: PORTAL_API_KEY was not defined, disabling Jupyter workflow') + self.warning( + '_get_jupyter_runid: PORTAL_API_KEY was not defined, disabling Jupyter workflow' + ) self._portal_runid = -2 self._portal_runid_event.set() return @@ -2156,17 +2380,22 @@ def _establish_portal_runid(self) -> None: except Exception: attempts += 1 if attempts >= max_attempts: - self.warning('_get_jupyter_runid: Unable to get RUNID directly from remote portal, disabling Jupyter workflow') + self.warning( + '_get_jupyter_runid: Unable to get RUNID directly from remote portal, disabling Jupyter workflow' + ) self._portal_runid = -2 self._portal_runid_event.set() - self.warning('took this amount of time to reach max attempts: %d', time.time() - base_time) + self.warning( + 'took this amount of time to reach max attempts: %d', + time.time() - base_time, + ) return self._portal_runid_event.wait(1.0) def initialize_jupyter_notebook( self, source_notebook_path: str, - dest_notebook_name: Optional[str] = None, + dest_notebook_name: str | None = None, ) -> None: """If the IPS Portal is available, this function loads a notebook from source_notebook_path, adds a cell to load the data, and then saves the concatenated notebook to the Portal. @@ -2180,7 +2409,7 @@ def initialize_jupyter_notebook( # have we initialized a runid yet? if not, block this function call until we can establish or not establish one if not self._portal_runid_event.is_set(): self._establish_portal_runid() - + # now check to see if we have a valid runid if self._portal_runid < 0: return @@ -2202,7 +2431,11 @@ def initialize_jupyter_notebook( portal_data: dict[str, Any] = {} portal_data['eventtype'] = 'PORTAL_REGISTER_NOTEBOOK' - portal_data['data_source'] = os.path.join(os.getcwd(), source_notebook_path) if not os.path.isabs(source_notebook_path) else source_notebook_path + portal_data['data_source'] = ( + os.path.join(os.getcwd(), source_notebook_path) + if not os.path.isabs(source_notebook_path) + else source_notebook_path + ) portal_data['username'] = self.get_config_param('USER') portal_data['filename'] = dest_notebook_name portal_data['portal_runid'] = self._portal_runid @@ -2210,7 +2443,9 @@ def initialize_jupyter_notebook( self.publish('_IPS_MONITOR', 'PORTAL_REGISTER_NOTEBOOK', event_data) self._send_monitor_event('IPS_PORTAL_REGISTER_NOTEBOOK', f'FILENAME = {dest_notebook_name}') - def add_analysis_data_files(self, current_data_file_paths: list[str], timestamp: float = 0.0, replace: bool = False) -> None: + def add_analysis_data_files( + self, current_data_file_paths: list[str], timestamp: float = 0.0, replace: bool = False + ) -> None: """If the IPS Portal is available, saves data files to IPS Portal. Files are indexed via specific timestamps. If a connection to the IPS Portal cannot be verified for this run, this function does nothing. @@ -2241,7 +2476,9 @@ def add_analysis_data_files(self, current_data_file_paths: list[str], timestamp: portal_data: dict[str, Any] = {} portal_data['eventtype'] = 'PORTAL_ADD_JUPYTER_DATA' - portal_data['data_source'] = os.path.join(os.getcwd(), source) if not os.path.isabs(source) else source + portal_data['data_source'] = ( + os.path.join(os.getcwd(), source) if not os.path.isabs(source) else source + ) portal_data['username'] = self.get_config_param('USER') portal_data['filename'] = filename portal_data['tag'] = timestamp @@ -2249,44 +2486,47 @@ def add_analysis_data_files(self, current_data_file_paths: list[str], timestamp: portal_data['portal_runid'] = self._portal_runid event_data['portal_data'] = portal_data self.publish('_IPS_MONITOR', 'PORTAL_ADD_JUPYTER_DATA', event_data) - self._send_monitor_event('IPS_PORTAL_ADD_JUPYTER_DATA', f'SOURCE = {source} TIMESTAMP = {timestamp} REPLACE = {replace}') + self._send_monitor_event( + 'IPS_PORTAL_ADD_JUPYTER_DATA', + f'SOURCE = {source} TIMESTAMP = {timestamp} REPLACE = {replace}', + ) - def publish(self, topicName: str, eventName: str, eventBody: Any) -> None: + def publish(self, topic_name: str, event_name: str, event_body: Any) -> None: """ - Publish event consisting of *eventName* and *eventBody* to topic *topicName* to the IPS event service. + Publish event consisting of *event_name* and *event_body* to topic *topic_name* to the IPS event service. Publishing an event multiple components are subscribed to will cause each component to handle the message simultaneously. - :param topicName: the name of the topic to publish on, top-level namespace - :param eventName: event associated with the topic - :param eventBody: data to send + :param topic_name: the name of the topic to publish on, top-level namespace + :param event_name: event associated with the topic + :param event_body: data to send """ - if not topicName.startswith('_IPS'): - topicName = self.sim_name + '_' + topicName - self.event_service.publish(topicName, eventName, eventBody) + if not topic_name.startswith('_IPS'): + topic_name = self.sim_name + '_' + topic_name + self.event_service.publish(topic_name, event_name, event_body) - def subscribe(self, topicName: str, callback: Callable) -> None: + def subscribe(self, topic_name: str, callback: Callable) -> None: """ - Subscribe to topic *topicName* on the IPS event service and register *callback* as the method to be invoked when an event is published to that topic. + Subscribe to topic *topic_name* on the IPS event service and register *callback* as the method to be invoked when an event is published to that topic. Multiple components can subscribe to the same topic name; if this is the case, each component will handle the message separately when the topic is published to. - - :param topicName: the name of the topic to subscribe to, top-level namespace + + :param topic_name: the name of the topic to subscribe to, top-level namespace :param callback: the function which will be called on receiving a message """ - if not topicName.startswith('_IPS'): - topicName = self.sim_name + '_' + topicName - self.event_service.subscribe(topicName, callback) + if not topic_name.startswith('_IPS'): + topic_name = self.sim_name + '_' + topic_name + self.event_service.subscribe(topic_name, callback) - def unsubscribe(self, topicName: str) -> None: + def unsubscribe(self, topic_name: str) -> None: """ - Remove subscription to topic *topicName*. + Remove subscription to topic *topic_name*. - :param topicName: the name of the topic to unsubscribe from + :param topic_name: the name of the topic to unsubscribe from """ - if not topicName.startswith('_IPS'): - topicName = self.sim_name + '_' + topicName - self.event_service.unsubscribe(topicName) + if not topic_name.startswith('_IPS'): + topic_name = self.sim_name + '_' + topic_name + self.event_service.unsubscribe(topic_name) def process_events(self) -> None: """ @@ -2298,13 +2538,18 @@ def send_portal_event( self, event_type: IPS_EVENT_TYPE = 'COMPONENT_EVENT', event_comment: str = '', - event_time: Optional[float] = None, - elapsed_time: Optional[float] = None, + event_time: float | None = None, + elapsed_time: float | None = None, ): """ Send event to web portal. """ - return self._send_monitor_event(eventType=event_type, comment=event_comment, event_time=event_time, elapsed_time=elapsed_time) + return self._send_monitor_event( + event_type=event_type, + comment=event_comment, + event_time=event_time, + elapsed_time=elapsed_time, + ) def log(self, msg, *args): """ @@ -2356,7 +2601,16 @@ def create_task_pool(self, task_pool_name: str): raise Exception('Error: Duplicate task pool name %s' % (task_pool_name)) self.task_pools[task_pool_name] = TaskPool(task_pool_name, self) - def add_task(self, task_pool_name: str, task_name: str, nproc: int, working_dir: str, binary: str, *args, **keywords): + def add_task( + self, + task_pool_name: str, + task_name: str, + nproc: int, + working_dir: str, + binary: str, + *args, + **keywords, + ): """ Add task *task_name* to task pool *task_pool_name*. Remaining arguments are the same as in :py:meth:`ServicesProxy.launch_task`. @@ -2415,20 +2669,29 @@ def submit_tasks( :returns: task return value """ start_time = time.time() - self._send_monitor_event('IPS_TASK_POOL_BEGIN', - 'task_pool = %s ' % task_pool_name) + self._send_monitor_event('IPS_TASK_POOL_BEGIN', 'task_pool = %s ' % task_pool_name) task_pool: TaskPool = self.task_pools[task_pool_name] retval = task_pool.submit_tasks( - block, use_dask, dask_nodes, dask_ppw, launch_interval, - use_shifter, shifter_args, dask_worker_plugin, - dask_worker_per_gpu, oversubscribe, hwthreads, - logfile, errfile + block, + use_dask, + dask_nodes, + dask_ppw, + launch_interval, + use_shifter, + shifter_args, + dask_worker_plugin, + dask_worker_per_gpu, + oversubscribe, + hwthreads, + logfile, + errfile, ) elapsed_time = time.time() - start_time - self._send_monitor_event('IPS_TASK_POOL_END', - 'task_pool = %s elapsed time = %.2f S' % ( - task_pool_name, elapsed_time), - elapsed_time=elapsed_time) + self._send_monitor_event( + 'IPS_TASK_POOL_END', + 'task_pool = %s elapsed time = %.2f S' % (task_pool_name, elapsed_time), + elapsed_time=elapsed_time, + ) return retval def get_finished_tasks(self, task_pool_name: str): @@ -2446,7 +2709,9 @@ def remove_task_pool(self, task_pool_name: str): task_pool.terminate_tasks() del self.task_pools[task_pool_name] - def create_sub_workflow(self, sub_name, config_file, override: Optional[dict[str, Any]] = None, input_dir=None): + def create_sub_workflow( + self, sub_name, config_file, override: dict[str, Any] | None = None, input_dir=None + ): """Create sub-workflow :param sub_name: name of sub-workflow @@ -2531,7 +2796,9 @@ def create_sub_workflow(self, sub_name, config_file, override: Optional[dict[str sub_conf_new.filename = os.path.basename(config_file) sub_conf_new.write() try: # FIXME, if you're going to catch an exception, you should handle it - (sim_name, init_comp, driver_comp) = self._create_simulation(os.path.abspath(sub_conf_new.filename), {}, sub_workflow=True) + (sim_name, init_comp, driver_comp) = self._create_simulation( + os.path.abspath(sub_conf_new.filename), {}, sub_workflow=True + ) except Exception: raise @@ -2551,7 +2818,9 @@ def _create_simulation(self, config_file, override, sub_workflow=False): :returns: tuple of simulation name, init component, driver component """ try: - msg_id = self._invoke_service(self.fwk.component_id, 'create_simulation', config_file, override, sub_workflow) + msg_id = self._invoke_service( + self.fwk.component_id, 'create_simulation', config_file, override, sub_workflow + ) self.debug('create_simulation() msg_id = %s', msg_id) (sim_name, init_comp, driver_comp) = self._get_service_response(msg_id, block=True) self.debug('Created simulation %s', sim_name) @@ -2562,16 +2831,16 @@ def _create_simulation(self, config_file, override, sub_workflow=False): def run_ensemble( self, - template: Union[str, os.PathLike], + template: str | os.PathLike, variables: dict[str, dict[str, list[str]]], - run_dir: Union[str, os.PathLike], + run_dir: str | os.PathLike, name: str, - num_nodes: int = None, - cores_per_instance: Optional[int] = None, + num_nodes: int | None = None, + cores_per_instance: int | None = None, oversubscribe: bool = False, hwthreads: bool = False, - logfile: Union[str, os.PathLike] = None, - errfile: Union[str, os.PathLike] = None, + logfile: str | os.PathLike | None = None, + errfile: str | os.PathLike | None = None, ): """Run ensemble of simulations given the template and variables. @@ -2579,19 +2848,19 @@ def run_ensemble( .. code-block:: python - variables = {'a_sim_comp': {'A': [3, 2, 4], + variables = {'ASimComp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], 'C': ['bar', 'baz', 'quux']}, - 'another_sim_comp': {'D': [7, 5, 9], + 'AnotherSimComp': {'D': [7, 5, 9], 'B': [0.775, 0.080, 29.2], 'F': ['xyzzy', 'plud', 'thud']}} That is, the keys are the simulation names and the values are dicts mapping parameter to a set of values. Ensembles will be spun up for each simulation for each combination of parameters. E.g., - `a_sim_comp` will be run three times with the parameters of A, B, and C + `ASimComp` will be run three times with the parameters of A, B, and C being set to 3, 2.34, 'bar' for one of the simulation instances, - respectively. another_sim_comp behaves similarly with its + respectively. AnotherSimComp behaves similarly with its respective parameters. The ensembles will run under `run_dir` within a subdirectory @@ -2636,11 +2905,7 @@ def run_ensemble( self.debug(f'run_ensemble() use portal = {use_portal!s}') - def create_driver_config_file(template, - working_dir, - variables, - name, - use_portal): + def create_driver_config_file(template, working_dir, variables, name, use_portal): """Create an IPS config file for an ensemble instance :param template: ConfigObj from which to derive the config file @@ -2662,11 +2927,14 @@ def create_driver_config_file(template, template['_IPS_PORTAL_ENSEMBLE_ID'] = portal_ensemble_id template['SIM_NAME'] = name - if 'SIM_ROOT' in template and \ - template['SIM_ROOT'] is not None and \ - template['SIM_ROOT'].strip() != '': - self.info(f'SIM_ROOT in template config assigned a value, ' - f'{template["SIM_ROOT"]}, that will be ignored') + if ( + 'SIM_ROOT' in template + and template['SIM_ROOT'] is not None + and template['SIM_ROOT'].strip() != '' + ): + self.info( + f'SIM_ROOT in template config assigned a value, {template["SIM_ROOT"]}, that will be ignored' + ) # Ensure that the instance gets a unique directory for its work # by setting SIM_ROOT to the prefix path. @@ -2674,10 +2942,11 @@ def create_driver_config_file(template, # Handle portal configuration, note that PORTAL_API_KEY should be # an environment variable and will be passed in later. - self.debug(f'use_portal inside create_driver_config_file: ' - f'{use_portal}, with type {type(use_portal)}') + self.debug( + f'use_portal inside create_driver_config_file: {use_portal}, with type {type(use_portal)}' + ) if use_portal: - self.debug(f'USE_PORTAL is True, so emitting PORTAL variables.') + self.debug('USE_PORTAL is True, so emitting PORTAL variables.') # WARNING: portal_runid is set asynchronously by the Portal # Bridge, wait for it to be set currently, the value we use # in the config file is the value the Bridge component itself @@ -2688,19 +2957,18 @@ def create_driver_config_file(template, portal_runid = None while portal_runid is None: self.debug('Attempting to get portal run ID') - portal_runid = self.get_config_param('PORTAL_RUNID', - silent=True) + portal_runid = self.get_config_param('PORTAL_RUNID', silent=True) self.debug(f'Using portal run ID: {portal_runid}') - portal_url = self.get_config_param('PORTAL_URL', - silent=True) + portal_url = self.get_config_param('PORTAL_URL', silent=True) template['PORTAL_URL'] = portal_url template['USE_PORTAL'] = 'True' template['PARENT_PORTAL_RUNID'] = portal_runid else: - self.debug('USE_PORTAL is False, so propagating that to ' - 'ensemble instance config file.') + self.debug( + 'USE_PORTAL is False, so propagating that to ensemble instance config file.' + ) template['USE_PORTAL'] = 'False' # We need to plug in the variables, so we need to find the section @@ -2803,7 +3071,7 @@ def send_ensemble_instance_to_portal(ensemble_name: str, data_path: Path) -> Non # have we initialized a runid yet? if not, block this function call until we can establish or not establish one if not self._portal_runid_event.is_set(): self._establish_portal_runid() - + # now check to see if we have a valid runid if self._portal_runid < 0: return @@ -2824,14 +3092,16 @@ def send_ensemble_instance_to_portal(ensemble_name: str, data_path: Path) -> Non self.publish('_IPS_MONITOR', 'PORTAL_UPLOAD_ENSEMBLE_PARAMS', event_data) self._send_monitor_event('IPS_PORTAL_UPLOAD_ENSEMBLE_PARAMS', f'NAME = {name}') - + # wait on a confirmation from the portal before proceeding with launching the ensembles ensemble_portal_wait_time = 10.0 while not self._ensemble_verification_check: time.sleep(1.0) ensemble_portal_wait_time -= 1.0 if ensemble_portal_wait_time <= 0.0: - self.warning('Could not confirm ensemble parameters upload, proceeding with task submission anyway') + self.warning( + 'Could not confirm ensemble parameters upload, proceeding with task submission anyway' + ) break self.process_events() self._ensemble_verification_check = False # reset in case run_ensemble is called again @@ -2847,7 +3117,6 @@ def send_ensemble_instance_to_portal(ensemble_name: str, data_path: Path) -> Non num_nodes = 1 self.info(f'run_ensemble() num_nodes = {num_nodes}') - # Ensure that we create a unique task pool name for this using the # instance prefix `name` # check this first to ensure uniqueness of `name` parameter @@ -2865,7 +3134,7 @@ def send_ensemble_instance_to_portal(ensemble_name: str, data_path: Path) -> Non # Let's first "flatten" the hierarchical variables dict into a list # of lists of dicts, where the top-level of which contains the ensemble # instance name and associated parameters. - self.debug(f'Grouping variables into instances') + self.debug('Grouping variables into instances') instances = ipsutil.group_ensemble_variables_into_instances(variables, name) # save the variables on both disk and to the IPS Portal @@ -2899,34 +3168,35 @@ def send_ensemble_instance_to_portal(ensemble_name: str, data_path: Path) -> Non # instance, particularly because part of the error checking is to # ensure that all the variables have been assigned. The first # instance element contains the ensemble instance name. - simulation_filename = create_driver_config_file(deepcopy(template_config), - working_dir, - instance[1], - instance[0], - use_portal) - self.debug(f'Simulation config file for instance {instance[0]} is ' - f'{simulation_filename}') + simulation_filename = create_driver_config_file( + deepcopy(template_config), working_dir, instance[1], instance[0], use_portal + ) + self.debug( + f'Simulation config file for instance {instance[0]} is {simulation_filename}' + ) # Create the bespoke platform config file for this instance - platform_filename = create_platform_config_file(instance[0], - working_dir, - cores_per_instance) - self.debug(f'Platform config file for instance {instance[0]} is ' - f'{platform_filename}') + platform_filename = create_platform_config_file( + instance[0], working_dir, cores_per_instance + ) + self.debug(f'Platform config file for instance {instance[0]} is {platform_filename}') # Submit a task to run the simulation instance, which is another # IPS run pointed to that config file. - args = [f'--simulation={simulation_filename}', f'--log={log_file}', f'--platform={platform_filename!s}'] + args = [ + f'--simulation={simulation_filename}', + f'--log={log_file}', + f'--platform={platform_filename!s}', + ] - kwargs = {} # optionally add logfile and errfile + kwargs = {} # optionally add logfile and errfile if logfile: kwargs['logfile'] = logfile if errfile: kwargs['errfile'] = errfile - if self.fwk.logger.getEffectiveLevel() == logging.DEBUG: - self.debug(f'Setting subordinate instances logger to debug') + self.debug('Setting subordinate instances logger to debug') # If we're in debug mode, then also pass the debug flag. # May as well pass in the --verbose, too. args.insert(1, '--debug') @@ -2939,7 +3209,7 @@ def send_ensemble_instance_to_portal(ensemble_name: str, data_path: Path) -> Non self.logger.debug(f'Submitting {len(instances)} ensemble tasks') num_submitted = self.submit_tasks( task_pool_name, - block=True, + block=True, use_dask=True, dask_nodes=num_nodes, dask_ppw=cores_per_instance, @@ -2992,12 +3262,10 @@ def setup(self, worker: Worker): """ if 'HWLOC_XMLFILE' in os.environ: # Remove HWLOC_XMLFILE to avoid issues with OpenMPI on Dask workers - self.logger.debug('Removing HWLOC_XMLFILE environment variable for ' - 'Dask worker') + self.logger.debug('Removing HWLOC_XMLFILE environment variable for Dask worker') del os.environ['HWLOC_XMLFILE'] else: - self.logger.debug('HWLOC_XMLFILE environment variable not set ' - 'for Dask worker') + self.logger.debug('HWLOC_XMLFILE environment variable not set for Dask worker') # Necessary to ensure the DVM "sees" all the resources to manage os.environ['PRTE_MCA_ras_slurm_use_entire_allocation'] = '1' @@ -3007,9 +3275,11 @@ def setup(self, worker: Worker): self.logger.info('Launching DVM') self.worker.dvm_uri_file = f'/tmp/dvm.uri.{os.getpid()}' - command = [#'srun', '--mpi=pmix_v4', '-N', os.environ['SLURM_NNODES'], '--ntasks-per-node=1', - 'prte', #'--no-daemonize', - '--report-uri', self.worker.dvm_uri_file] + command = [ #'srun', '--mpi=pmix_v4', '-N', os.environ['SLURM_NNODES'], '--ntasks-per-node=1', + 'prte', #'--no-daemonize', + '--report-uri', + self.worker.dvm_uri_file, + ] mapping_policy = 'core' # by default bind to cores if self.hwthreads: @@ -3026,9 +3296,9 @@ def setup(self, worker: Worker): try: self.logger.debug(f'Executing command: {command!s}') - self.worker.dvm_proc = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + self.worker.dvm_proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) except Exception as e: print(f'Exception during setting up DVM: {e}') console.print(Traceback.from_exception(type(e), e, e.__traceback__)) @@ -3051,7 +3321,6 @@ def setup(self, worker: Worker): os.environ['PMIX_SERVER_URI41'] = self.worker.dvm_uri - def teardown(self, worker: Worker): self.logger.info(f'Shutting down DVM at {self.worker.dvm_uri}') @@ -3093,7 +3362,7 @@ class TaskPool: # scheduler` and `dask worker` dask = shutil.which('dask') dask_scheduler = [dask, 'scheduler'] - dask_worker = [dask, 'worker'] + DaskWorker = [dask, 'worker'] shifter = shutil.which('shifter') @@ -3150,7 +3419,9 @@ def _wait_active_tasks(self): while len(self.active_tasks) > 0: self._wait_any_task() - def add_task(self, task_name: str, nproc: int, working_dir: str, binary: str, *args, **keywords): + def add_task( + self, task_name: str, nproc: int, working_dir: str, binary: str, *args, **keywords + ): """ Create :py:obj:`Task` object and add to *queued_tasks* of the task pool. Raise exception if task name already exists in task pool. @@ -3186,7 +3457,9 @@ def add_task(self, task_name: str, nproc: int, working_dir: str, binary: str, *a keywords['keywords']['block'] = False self.serial_pool = self.serial_pool and (nproc == 1) - self.queued_tasks[task_name] = Task(task_name, nproc, working_dir, binary_fullpath, *args, **keywords['keywords']) + self.queued_tasks[task_name] = Task( + task_name, nproc, working_dir, binary_fullpath, *args, **keywords['keywords'] + ) @staticmethod def _launch_keywords_with_defaults(task_keywords, logfile=None, errfile=None): @@ -3199,7 +3472,7 @@ def _launch_keywords_with_defaults(task_keywords, logfile=None, errfile=None): return keywords def _process_dask_event(self, event): - """ This will create an IPS monitor event from a Dask event + """This will create an IPS monitor event from a Dask event These events will have been created in `launch()`. As they are created, this callback will be invoked to send the corresponding @@ -3211,8 +3484,7 @@ def _process_dask_event(self, event): """ timestamp, message = event - self.services.debug(f'Processing dask event: {message!s}, ' - f'timestamp: {timestamp!s}') + self.services.debug(f'Processing dask event: {message!s}, timestamp: {timestamp!s}') if 'worker' in message: # Sneaky Dask will surreptitiously add 'worker', which is @@ -3223,10 +3495,9 @@ def _process_dask_event(self, event): self.services._send_monitor_event(**message) - def submit_dask_tasks( self, - block= True, + block=True, dask_nodes=1, dask_ppw=None, use_shifter=False, @@ -3280,7 +3551,9 @@ def submit_dask_tasks( :returns: number of tasks submitted """ - def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shifter_args=None): + def _make_worker_args( + num_workers: int, num_threads: int, use_shifter: bool, shifter_args=None + ): """Make Dask worker command line arguments. :param num_workers: Number of workers to start @@ -3290,7 +3563,7 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi to start a Dask worker """ base_args = [ - *self.dask_worker, + *self.DaskWorker, '--no-dashboard', '--no-nanny', '--scheduler-file', @@ -3317,7 +3590,10 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi # Note that we use the absolute path since at some point we may # be in a different directory, which means that we otherwise would # not be able to find the Dask scheduler file. - self.dask_scheduler_file = Path('.').absolute() / f'{self.name}_dask_sched_{datetime.now().strftime("%Y%m%d%S")}.json' + self.dask_scheduler_file = ( + Path('.').absolute() + / f'{self.name}_dask_sched_{datetime.now().strftime("%Y%m%d%S")}.json' + ) # self.dask_scheduler_file = os.path.join(os.getcwd(), f'{self.name}_dask_shed_{time.time()}.json') if use_shifter: @@ -3357,28 +3633,26 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi ) self.dask_sched_pid = self.dask_sched_popen.pid - else: # We are NOT using shifter + else: # We are NOT using shifter try: args = [ - *self.dask_scheduler, - '--no-dashboard', - '--no-jupyter', - '--no-show', - '--idle-timeout', - str(TaskPool.IDLE_TIMEOUT), - '--scheduler-file', - str(self.dask_scheduler_file), - '--port', - '0', - ] + *self.dask_scheduler, + '--no-dashboard', + '--no-jupyter', + '--no-show', + '--idle-timeout', + str(TaskPool.IDLE_TIMEOUT), + '--scheduler-file', + str(self.dask_scheduler_file), + '--port', + '0', + ] self.services.info(f'Scheduler args: {" ".join(args)}') self.dask_sched_popen = subprocess.Popen(args) self.dask_sched_pid = self.dask_sched_popen.pid - self.services.info(f'Scheduler pid: ' - f'{self.dask_sched_popen.pid}') + self.services.info(f'Scheduler pid: {self.dask_sched_popen.pid}') except Exception as e: - self.services.critical(f'Exception while starting Dask ' - f'scheduler: {e!s}') + self.services.critical(f'Exception while starting Dask scheduler: {e!s}') console.print_exception(show_locals=True) # TODO better error handling than just re-raising the exception raise @@ -3416,8 +3690,12 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi cores_per_node = services.get_config_param('CORES_PER_NODE') if dask_ppw is not None: - self.services.debug(f'Using {dask_ppw} processes per Dask worker via dask_ppw argument') - print(f'Using {dask_ppw} processes per Dask worker via dask_ppw argument', flush=True) + self.services.debug( + f'Using {dask_ppw} processes per Dask worker via dask_ppw argument' + ) + print( + f'Using {dask_ppw} processes per Dask worker via dask_ppw argument', flush=True + ) nthreads = cores_per_node // dask_ppw else: nthreads = cores_per_node @@ -3437,22 +3715,27 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi print(f'Using {dask_ppw} processes per Dask worker via dask_ppw argument', flush=True) else: dask_ppw = int(services.get_config_param('PROCS_PER_NODE')) - self.services.debug(f'using {services.get_config_param("PROCS_PER_NODE")} processes per Dask worker from platform config PROCS_PER_NODE') + self.services.debug( + f'using {services.get_config_param("PROCS_PER_NODE")} processes per Dask worker from platform config PROCS_PER_NODE' + ) self.services.info(f'Threads per Dask worker is {nthreads}') # --nprocs was removed in version 2022.10.0 and replaced with --nworkers # nworkers = '--nworkers' if tuple(map(int, self.distributed.__version__.split('.'))) >= (2022, 10, 0) else '--nprocs' - workers_cmd_line = _make_worker_args(num_workers=1, num_threads=nthreads, use_shifter=use_shifter, shifter_args=shifter_args) + workers_cmd_line = _make_worker_args( + num_workers=1, num_threads=nthreads, use_shifter=use_shifter, shifter_args=shifter_args + ) self.services.debug(f'Dask workers command line: {workers_cmd_line}') - self.dask_workers_tid = services.launch_task(dask_nodes, os.getcwd(), *workers_cmd_line, task_ppn=task_ppn, task_gpp=task_gpp) + self.dask_workers_tid = services.launch_task( + dask_nodes, os.getcwd(), *workers_cmd_line, task_ppn=task_ppn, task_gpp=task_gpp + ) self.services.debug(f'Dask scheduler pid: {self.dask_sched_popen.pid}') - self.dask_client = Client(scheduler_file=self.dask_scheduler_file, - direct_to_workers=True) + self.dask_client = Client(scheduler_file=self.dask_scheduler_file, direct_to_workers=True) self.services.debug(f'Dask client: {self.dask_client!s}') # And logging done via the dask workers will be forwarded to the root @@ -3473,9 +3756,9 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi # Regardless of any other worker plugins, we need this plugin to setup # the DVM for the workers so that OpenMPI can work properly. self.services.debug('Registering DVMPlugin') - self.dask_client.register_plugin(DVMPlugin(logger=services.logger, - oversubscribe=oversubscribe, - hwthreads=hwthreads)) + self.dask_client.register_plugin( + DVMPlugin(logger=services.logger, oversubscribe=oversubscribe, hwthreads=hwthreads) + ) self.services.debug('Registered DVMPlugin') # Wait for so many workers to be online before proceeding to @@ -3491,11 +3774,16 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi else: self.services.info('Only a single Dask worker needed, proceeding') - try: # FIXME this is deprecated, but be mindful of blithely deleting - file_id = str(self.services._portal_runid) if self.services._portal_runid > 0 else self.services._fallback_portal_runid - self.worker_event_logfile = services.sim_name + '_' + file_id + '_' + self.name + '_{}.json' + file_id = ( + str(self.services._portal_runid) + if self.services._portal_runid > 0 + else self.services._fallback_portal_runid + ) + self.worker_event_logfile = ( + services.sim_name + '_' + file_id + '_' + self.name + '_{}.json' + ) self.services.debug(f'Worker event log file: {self.worker_event_logfile}') except KeyError: # USE_PORTAL == False @@ -3513,12 +3801,12 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi cpus_per_procs = [] worker_event_logfiles = [] for task_name, task in self.queued_tasks.items(): - self.services.debug(f'Submitting task {task_name} to dask client with {dask_ppw} cores per worker') + self.services.debug( + f'Submitting task {task_name} to dask client with {dask_ppw} cores per worker' + ) self.services.debug(f'Task {task_name} working dir: {task.working_dir}') self.services.debug(f'Task args: {task.args} keywords: {task.keywords}') - keywords = self._launch_keywords_with_defaults(task.keywords, - logfile, - errfile) + keywords = self._launch_keywords_with_defaults(task.keywords, logfile, errfile) task_names.append(task_name) binaries.append(task.binary) working_dirs.append(task.working_dir) @@ -3543,13 +3831,11 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi self.queued_tasks = {} if block: - self.services.debug(f'submit_dask_tasks: blocking tasks to await ' - f'results') + self.services.debug('submit_dask_tasks: blocking tasks to await results') # Await all the futures to finish, thereby blocking until they # are all done. result = self.dask_client.gather(self.futures, direct=True) - self.services.debug(f'submit_dask_tasks: have {len(result)} ' - f'results, block released') + self.services.debug(f'submit_dask_tasks: have {len(result)} results, block released') # TODO check actual result values for problems # Set this to empty list so that get_dask_finished_tasks_status @@ -3559,7 +3845,7 @@ def _make_worker_args(num_workers: int, num_threads: int, use_shifter: bool, shi # Since we're done with Dask, let's shut it down self._shutdown_dask() else: - self.services.debug(f'submit_dask_tasks: not blocking tasks') + self.services.debug('submit_dask_tasks: not blocking tasks') return len(self.futures) @@ -3626,24 +3912,32 @@ def submit_tasks( if TaskPool.dask and TaskPool.distributed and self.serial_pool: self.dask_pool = True if use_shifter and not self.shifter: - self.services.error('Requested to run dask within shifter ' - 'but shifter not available') + self.services.error( + 'Requested to run dask within shifter but shifter not available' + ) raise RuntimeError('shifter not found') else: return self.submit_dask_tasks( - block, dask_nodes, dask_ppw, use_shifter, - shifter_args, dask_worker_plugin, - dask_worker_per_gpu, oversubscribe, hwthreads, - logfile, errfile + block, + dask_nodes, + dask_ppw, + use_shifter, + shifter_args, + dask_worker_plugin, + dask_worker_per_gpu, + oversubscribe, + hwthreads, + logfile, + errfile, ) elif not TaskPool.dask or not TaskPool.distributed: raise RuntimeError( - 'Requested use_dask but cannot because import dask or ' - 'distributed failed') + 'Requested use_dask but cannot because import dask or distributed failed' + ) elif not self.serial_pool: self.services.warning( - 'Requested use_dask but cannot because multiple ' - 'processors requested') + 'Requested use_dask but cannot because multiple processors requested' + ) submit_count = 0 # Make sure any finished tasks are handled before attempting to submit @@ -3652,8 +3946,7 @@ def submit_tasks( while True: if len(self.queued_tasks) == 0: break - active_tasks = self.services.launch_task_pool(self.name, - launch_interval) + active_tasks = self.services.launch_task_pool(self.name, launch_interval) for task_name, task_id in active_tasks.items(): self.active_tasks[task_id] = self.queued_tasks.pop(task_name) submit_count += 1 @@ -3684,7 +3977,7 @@ def _shutdown_dask(self): if self.dask_client is not None: # Shutdown handles ending client, scheduler, and workers - self.dask_client.unsubscribe_topic('ips') # unregister handler + self.dask_client.unsubscribe_topic('ips') # unregister handler self.services.debug('Unsubscribed from Dask client ips topic') self.dask_client.shutdown() self.services.debug('Shutdown Dask client') @@ -3717,7 +4010,7 @@ def _shutdown_dask(self): self.services.wait_task(self.dask_workers_tid) self.dask_scheduler_file = None self.dask_workers_tid = None - self.dask_sched_pid: Optional[int] = None + self.dask_sched_pid: int | None = None self.dask_sched_popen = None self.dask_pool = False @@ -3727,7 +4020,6 @@ def _shutdown_dask(self): self.services.debug('Shutdown Dask system') - def get_dask_finished_tasks_status(self): """Return a dictionary of exit status values for all dask tasks that have finished since the last time finished tasks were polled. @@ -3786,26 +4078,21 @@ def get_dask_finished_tasks_status(self): self.services.debug('get_dask_finished_tasks_status: after gather()') # If we don't have a result, then there were no tasks to gather. if result is None: - self.services.warning( - 'No futures available in call to finished ') + self.services.warning('No futures available in call to finished ') self._shutdown_dask() return {} else: - self.services.debug( - f'get_dask_finished_tasks_status: have {len(result)} futures') + self.services.debug(f'get_dask_finished_tasks_status: have {len(result)} futures') else: # This is ok if submit_dask_tasks.block = True, but we echo this # anyway in debug mode as a reality check. self.services.debug('get_dask_finished_tasks_status: have no futures') - # NOTE: You may get an exception stack trace from Dask, this is currently not believed to cause an issue. # We no longer need Dask running, so shut it down. - self.services.debug(f'get_dask_finished_tasks_status: before _shutdown_dask()') + self.services.debug('get_dask_finished_tasks_status: before _shutdown_dask()') self._shutdown_dask() - self.services.debug(f'get_dask_finished_tasks_status: after _shutdown_dask()') - - + self.services.debug('get_dask_finished_tasks_status: after _shutdown_dask()') if result is not None: self.services.debug('get_dask_finished_tasks_status: have result') @@ -3813,8 +4100,7 @@ def get_dask_finished_tasks_status(self): # is doubtful. return dict(result) - self.services.debug('get_dask_finished_tasks_status: no result, ' - 'returning None') + self.services.debug('get_dask_finished_tasks_status: no result, returning None') return result # which will be none or {} @@ -3872,7 +4158,9 @@ class Task: :param \*\*keywords: keyword arguments for launching the task. See :py:meth:`ServicesProxy.launch_task` for details. """ - def __init__(self, task_name: str, nproc: int, working_dir: str, binary: str, *args, **keywords): + def __init__( + self, task_name: str, nproc: int, working_dir: str, binary: str, *args, **keywords + ): self.name = task_name self.nproc = int(nproc) self.working_dir = working_dir diff --git a/ipsframework/taskManager.py b/ipsframework/task_manager.py similarity index 66% rename from ipsframework/taskManager.py rename to ipsframework/task_manager.py index 92c55aef..a843cd15 100644 --- a/ipsframework/taskManager.py +++ b/ipsframework/task_manager.py @@ -5,21 +5,34 @@ import sys from collections import namedtuple from math import ceil -from typing import List - -from . import configurationManager, messages -from .ipsExceptions import ( - BadResourceRequestException, - BlockedMessageException, - GPUResourceRequestMismatchException, - IncompleteCallException, - InsufficientResourcesException, - ResourceRequestMismatchException, + +from . import messages +from .ips_exceptions import ( + BadResourceRequestError, + BlockedMessageError, + GpuResourceRequestMismatchError, + IncompleteCallError, + InsufficientResourcesError, + ResourceRequestMismatchError, ) from .ipsutil import which TaskInit = namedtuple( - 'TaskInit', ['nproc', 'binary', 'working_dir', 'tppn', 'tcpp', 'tgpp', 'block', 'omp', 'wnodes', 'wsocks', 'cmd_args', 'launch_cmd_extra_args'] + 'TaskInit', + [ + 'nproc', + 'binary', + 'working_dir', + 'tppn', + 'tcpp', + 'tgpp', + 'block', + 'omp', + 'wnodes', + 'wsocks', + 'cmd_args', + 'launch_cmd_extra_args', + ], ) @@ -39,7 +52,7 @@ def __init__(self, fwk): self.resource_mgr = None self.config_mgr = None self.host = None - self.comp_registry = configurationManager.ComponentRegistry() + self.comp_registry = self.fwk.config_manager.comp_registry self.service_methods = [ 'init_call', 'launch_task', @@ -80,7 +93,7 @@ def initialize(self, data_mgr, resource_mgr, config_mgr): Initialize references to other managers and key values from configuration manager. """ - self.event_mgr = None # eventManager(self) + self.event_mgr = None # EventManager(self) self.data_mgr = data_mgr self.resource_mgr = resource_mgr self.config_mgr = config_mgr @@ -110,7 +123,7 @@ def get_task_id(self): self.next_task_id = self.next_task_id + 1 return retval - def printCurrTaskTable(self): + def print_curr_task_table(self): """ Prints the task table pretty-like. """ @@ -142,9 +155,22 @@ def init_call(self, init_call_msg, manage_return=True): keywords = init_call_msg.keywords caller_id = init_call_msg.sender_id call_id = self.get_call_id() - self.fwk.debug('TM:init_call(): %s %s %s %s', caller_id, callee_id, method_name, str(args)) - invoke_msg = messages.MethodInvokeMessage(self.fwk.component_id, callee_id, call_id, method_name, *args, **keywords) - invocation_q = self.comp_registry.getComponentArtifact(callee_id, 'invocation_q') + self.fwk.debug( + 'TM:init_call(): %s %s %s %s', + caller_id, + callee_id, + method_name, + str(args), + ) + invoke_msg = messages.MethodInvokeMessage( + self.fwk.component_id, + callee_id, + call_id, + method_name, + *args, + **keywords, + ) + invocation_q = self.comp_registry.get_component_artifact(callee_id, 'invocation_q') invocation_q.put(invoke_msg) if manage_return: self.outstanding_calls[call_id] = (caller_id, None) @@ -188,9 +214,9 @@ def wait_call(self, wait_msg: messages.ServiceRequestMessage): else: return response_msg.args if not blocking: - raise IncompleteCallException(call_id) + raise IncompleteCallError(call_id) else: - raise BlockedMessageException(wait_msg, '***call %s not finished' % call_id) + raise BlockedMessageError(wait_msg, '***call %s not finished' % call_id) def init_task(self, init_task_msg: messages.ServiceRequestMessage): r""" @@ -198,7 +224,7 @@ def init_task(self, init_task_msg: messages.ServiceRequestMessage): launch command using the binary and arguments provided by the requesting component. Return launch command to component via :py:obj:`messages.ServiceResponseMessage`. Raise exception if task - can not be launched at this time (:py:exc:`ipsExceptions.BadResourceRequestException`, :py:exc:`ipsExceptions.InsufficientResourcesException`). + can not be launched at this time (:py:exc:`ipsExceptions.BadResourceRequestError`, :py:exc:`ipsExceptions.InsufficientResourcesError`). *init_task_msg* is expected to be of type :py:obj:`messages.ServiceRequestMessage` @@ -222,32 +248,40 @@ def init_task(self, init_task_msg: messages.ServiceRequestMessage): 7. \+ *cmd_args*: any arguments for the executable """ caller_id = init_task_msg.sender_id - taskInit = init_task_msg.args[0] + task_init = init_task_msg.args[0] try: return self._init_task( caller_id, - int(taskInit.nproc), - taskInit.binary, - taskInit.working_dir, - int(taskInit.tppn), - taskInit.tcpp, - taskInit.omp, - taskInit.tgpp, - taskInit.wnodes, - taskInit.wsocks, - taskInit.cmd_args, - taskInit.launch_cmd_extra_args, + int(task_init.nproc), + task_init.binary, + task_init.working_dir, + int(task_init.tppn), + task_init.tcpp, + task_init.omp, + task_init.tgpp, + task_init.wnodes, + task_init.wsocks, + task_init.cmd_args, + task_init.launch_cmd_extra_args, ) - except InsufficientResourcesException as e: - if taskInit.block: - raise BlockedMessageException(init_task_msg, '***%s waiting for %d resources' % (caller_id, taskInit.nproc)) from e + except InsufficientResourcesError as e: + if task_init.block: + raise BlockedMessageError( + init_task_msg, + '***%s waiting for %d resources' % (caller_id, task_init.nproc), + ) from e else: raise - except BadResourceRequestException as e: - self.fwk.error('There has been a fatal error, %s requested %d too many processors in task %d', caller_id, e.deficit, e.task_id) + except BadResourceRequestError as e: + self.fwk.error( + 'There has been a fatal error, %s requested %d too many processors in task %d', + caller_id, + e.deficit, + e.task_id, + ) raise - except ResourceRequestMismatchException as e: + except ResourceRequestMismatchError as e: self.fwk.error( 'There has been a fatal error, %s requested too few processors per node to launch task %d (requested: procs = %d, ppn = %d)', caller_id, @@ -256,7 +290,7 @@ def init_task(self, init_task_msg: messages.ServiceRequestMessage): e.ppn, ) raise - except GPUResourceRequestMismatchException as e: + except GpuResourceRequestMismatchError as e: self.fwk.error( 'There has been a fatal error, %s requested too many GPUs per node to launch task %d (requested: ppn = %d, gpp = %d)', caller_id, @@ -268,14 +302,37 @@ def init_task(self, init_task_msg: messages.ServiceRequestMessage): except Exception: raise - def _init_task(self, caller_id, nproc, binary, working_dir, tppn, tcpp, omp, tgpp, wnodes, wsocks, cmd_args, launch_cmd_extra_args): + def _init_task( + self, + caller_id, + nproc, + binary, + working_dir, + tppn, + tcpp, + omp, + tgpp, + wnodes, + wsocks, + cmd_args, + launch_cmd_extra_args, + ): # handle for task related things task_id = self.get_task_id() - allocation = self.resource_mgr.get_allocation(caller_id, nproc, task_id, wnodes, wsocks, task_ppn=tppn, task_cpp=tcpp, task_gpp=tgpp) + allocation = self.resource_mgr.get_allocation( + caller_id, + nproc, + task_id, + wnodes, + wsocks, + task_ppn=tppn, + task_cpp=tcpp, + task_gpp=tgpp, + ) self.fwk.debug('RM: get_allocation() returned %s', str(allocation)) - if allocation.partial_node or allocation.accurateNodes: + if allocation.partial_node or allocation.accurate_nodes: nodes = ','.join(allocation.nodelist) else: nodes = '' @@ -288,7 +345,7 @@ def _init_task(self, caller_id, nproc, binary, working_dir, tppn, tcpp, omp, tgp allocation.ppn, allocation.max_ppn, nodes, - allocation.accurateNodes, + allocation.accurate_nodes, allocation.partial_node, task_id, allocation.cpp, @@ -314,12 +371,12 @@ def build_launch_cmd( self, nproc: int, binary: str, - cmd_args: List[str], + cmd_args: list[str], working_dir, ppn: int, max_ppn: int, nodes: str, - accurateNodes: bool, + accurate_nodes: bool, partial_nodes: bool, task_id: int, cpp=0, @@ -338,8 +395,8 @@ def build_launch_cmd( * ppn - processes per node value to use * max_ppn - maximum possible ppn for this allocation * nodes - comma separated list of node ids - * accurateNodes - if ``True``, launch on nodes in *nodes*, otherwise the parallel launcher determines the process placement - * partial_nodes - if ``True`` and *accurateNodes* and *task_launch_cmd* == 'mpirun', + * accurate_nodes - if ``True``, launch on nodes in *nodes*, otherwise the parallel launcher determines the process placement + * partial_nodes - if ``True`` and *accurate_nodes* and *task_launch_cmd* == 'mpirun', a host file is created specifying the exact placement of processes on cores. * core_list - used for creating host file with process to core mappings """ @@ -384,14 +441,22 @@ def build_launch_cmd( # resources managed by DVM for this task as displayed in # detailed messages sent to stdout prior to running the # desired IPS task. - cmd = ' '.join([mpicmd, '--display', 'ALLOCATION,MAP-DEVEL,BINDINGS', nproc_flag, str(nproc)]) + cmd = ' '.join( + [ + mpicmd, + '--display', + 'ALLOCATION,MAP-DEVEL,BINDINGS', + nproc_flag, + str(nproc), + ] + ) else: cmd = ' '.join([mpicmd, nproc_flag, str(nproc), ppn_flag, str(ppn)]) cmd = f'{cmd} -x PYTHONPATH' # Propagate PYTHONPATH to compute nodes - if accurateNodes: + if accurate_nodes: cmd = f'{cmd} {host_select} {nodes}' elif version == 'SGI': - if accurateNodes: + if accurate_nodes: core_dict = {} ppn_groups = {} num_cores = self.resource_mgr.cores_per_socket @@ -420,7 +485,14 @@ def build_launch_cmd( env_update = {'MPI_DSM_CPULIST': ':'.join(envlets)} return cmd, env_update else: - cmd = ' '.join([self.task_launch_cmd, str(ppn), binary, ' '.join(cmd_args)]) + cmd = ' '.join( + [ + self.task_launch_cmd, + str(ppn), + binary, + ' '.join(cmd_args), + ] + ) # -------------------------------------- # mpiexec (MPICH variants) @@ -448,11 +520,28 @@ def build_launch_cmd( cmd = f'{self.task_launch_cmd} {config_option}' self.curr_task_table[task_id]['node_file'] = cfg_fname return cmd, env_update - elif accurateNodes: # Need to assign tasks to nodes explicitly + elif accurate_nodes: # Need to assign tasks to nodes explicitly host_select = '--host ' + nodes - cmd = ' '.join([self.task_launch_cmd, host_select, nproc_flag, str(nproc), ppn_flag, str(ppn)]) + cmd = ' '.join( + [ + self.task_launch_cmd, + host_select, + nproc_flag, + str(nproc), + ppn_flag, + str(ppn), + ] + ) else: - cmd = ' '.join([self.task_launch_cmd, nproc_flag, str(nproc), ppn_flag, str(ppn)]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nproc_flag, + str(nproc), + ppn_flag, + str(ppn), + ] + ) # ------------------------------------ # aprun (Cray parallel launch) # ------------------------------------ @@ -464,46 +553,109 @@ def build_launch_cmd( if self.host in ['hopper', 'edison']: num_numanodes = self.resource_mgr.sockets_per_node num_cores = self.resource_mgr.cores_per_node - if accurateNodes: + if accurate_nodes: nlist_flag = '-L' num_nodes = len(nodes.split(',')) - ppn = int(ceil(float(nproc) / num_nodes)) - per_numa = int(ceil(float(ppn) / num_numanodes)) + ppn = ceil(float(nproc) / num_nodes) + per_numa = ceil(float(ppn) / num_numanodes) if per_numa == num_cores / num_numanodes: - cmd = ' '.join([self.task_launch_cmd, nproc_flag, str(nproc), ppn_flag, str(ppn), nlist_flag, nodes]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nproc_flag, + str(nproc), + ppn_flag, + str(ppn), + nlist_flag, + nodes, + ] + ) else: if num_nodes > 1: ppn = per_numa * num_numanodes ppn = min(nproc, ppn) - cmd = ' '.join([self.task_launch_cmd, nproc_flag, str(nproc), ppn_flag, str(ppn), by_numanode_flag, str(per_numa), nlist_flag, nodes]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nproc_flag, + str(nproc), + ppn_flag, + str(ppn), + by_numanode_flag, + str(per_numa), + nlist_flag, + nodes, + ] + ) else: - num_nodes = int(ceil(float(nproc) / ppn)) - ppn = int(ceil(float(nproc) / num_nodes)) - per_numa = int(ceil(float(ppn) / num_numanodes)) - if per_numa == self.resource_mgr.cores_per_node / self.resource_mgr.sockets_per_node: - cmd = ' '.join([self.task_launch_cmd, nproc_flag, str(nproc), ppn_flag, str(ppn)]) + num_nodes = ceil(float(nproc) / ppn) + ppn = ceil(float(nproc) / num_nodes) + per_numa = ceil(float(ppn) / num_numanodes) + if ( + per_numa + == self.resource_mgr.cores_per_node / self.resource_mgr.sockets_per_node + ): + cmd = ' '.join( + [ + self.task_launch_cmd, + nproc_flag, + str(nproc), + ppn_flag, + str(ppn), + ] + ) else: if num_nodes > 1: ppn = per_numa * num_numanodes ppn = min(nproc, ppn) - cmd = ' '.join([self.task_launch_cmd, nproc_flag, str(nproc), ppn_flag, str(ppn), by_numanode_flag, str(per_numa)]) - elif accurateNodes: + cmd = ' '.join( + [ + self.task_launch_cmd, + nproc_flag, + str(nproc), + ppn_flag, + str(ppn), + by_numanode_flag, + str(per_numa), + ] + ) + elif accurate_nodes: nlist_flag = '-L' - cmd = ' '.join([self.task_launch_cmd, nproc_flag, str(nproc), ppn_flag, str(ppn), nlist_flag, nodes]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nproc_flag, + str(nproc), + ppn_flag, + str(ppn), + nlist_flag, + nodes, + ] + ) else: - cmd = ' '.join([self.task_launch_cmd, nproc_flag, str(nproc), cpu_assign_flag, '%d-%d' % (max_ppn - 1, max_ppn - int(ppn)), ppn_flag, str(ppn)]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nproc_flag, + str(nproc), + cpu_assign_flag, + '%d-%d' % (max_ppn - 1, max_ppn - int(ppn)), + ppn_flag, + str(ppn), + ] + ) # ------------------------------------ # numactl (single process launcher) # ------------------------------------ elif self.task_launch_cmd == 'numactl': - if accurateNodes and partial_nodes: + if accurate_nodes and partial_nodes: proc_flag = '--physcpubind=' procs = '' for p in core_list: procs = ','.join([k.split(':')[1] for k in p[1]]) proc_flag += procs else: - self.fwk.warning('numactl needs accurateNodes') + self.fwk.warning('numactl needs accurate_nodes') proc_flag = '' cmd = f'{self.task_launch_cmd} {proc_flag}' elif self.task_launch_cmd == 'srun': @@ -511,17 +663,52 @@ def build_launch_cmd( nnodes_flag = '-N' num_nodes = len(nodes.split(',')) if partial_nodes: - cmd = ' '.join([self.task_launch_cmd, nnodes_flag, str(num_nodes), nproc_flag, str(nproc)]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nnodes_flag, + str(num_nodes), + nproc_flag, + str(nproc), + ] + ) else: cpuptask_flag = '-c' cpubind_flag = '--threads-per-core=1 --cpu-bind=cores' if gpp: gpuflags = f'--gpus-per-task={gpp}' - cmd = ' '.join([self.task_launch_cmd, nnodes_flag, str(num_nodes), nproc_flag, str(nproc), cpuptask_flag, str(cpp), cpubind_flag, gpuflags]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nnodes_flag, + str(num_nodes), + nproc_flag, + str(nproc), + cpuptask_flag, + str(cpp), + cpubind_flag, + gpuflags, + ] + ) else: - cmd = ' '.join([self.task_launch_cmd, nnodes_flag, str(num_nodes), nproc_flag, str(nproc), cpuptask_flag, str(cpp), cpubind_flag]) + cmd = ' '.join( + [ + self.task_launch_cmd, + nnodes_flag, + str(num_nodes), + nproc_flag, + str(nproc), + cpuptask_flag, + str(cpp), + cpubind_flag, + ] + ) if omp: - env_update = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': str(cpp)} + env_update = { + 'OMP_PLACES': 'threads', + 'OMP_PROC_BIND': 'spread', + 'OMP_NUM_THREADS': str(cpp), + } else: self.fwk.error('invalid task launch command.') raise RuntimeError('invalid task launch command.') @@ -550,32 +737,37 @@ def init_task_pool(self, init_task_msg: messages.ServiceRequestMessage): ret_dict = {} for task_name in task_dict: # handle for task related things - taskInit = task_dict[task_name] + task_init = task_dict[task_name] try: ret_dict[task_name] = self._init_task( caller_id, - taskInit.nproc, - taskInit.binary, - taskInit.working_dir, - taskInit.tppn, - taskInit.tcpp, - taskInit.omp, - taskInit.tgpp, - taskInit.wnodes, - taskInit.wsocks, - taskInit.cmd_args, - taskInit.launch_cmd_extra_args, + task_init.nproc, + task_init.binary, + task_init.working_dir, + task_init.tppn, + task_init.tcpp, + task_init.omp, + task_init.tgpp, + task_init.wnodes, + task_init.wsocks, + task_init.cmd_args, + task_init.launch_cmd_extra_args, ) - except InsufficientResourcesException: + except InsufficientResourcesError: continue - except BadResourceRequestException as e: - self.fwk.error('There has been a fatal error, %s requested %d too many processors in task %d', caller_id, e.deficit, e.task_id) + except BadResourceRequestError as e: + self.fwk.error( + 'There has been a fatal error, %s requested %d too many processors in task %d', + caller_id, + e.deficit, + e.task_id, + ) for task_id, _, _, _ in ret_dict.values(): self.resource_mgr.release_allocation(task_id, -1) del self.curr_task_table[task_id] raise - except ResourceRequestMismatchException as e: + except ResourceRequestMismatchError as e: self.fwk.error( 'There has been a fatal error, %s requested too few processors per node to launch task %d (request: procs = %d, ppn = %d)', caller_id, @@ -587,7 +779,7 @@ def init_task_pool(self, init_task_msg: messages.ServiceRequestMessage): self.resource_mgr.release_allocation(task_id, -1) del self.curr_task_table[task_id] raise - except GPUResourceRequestMismatchException as e: + except GpuResourceRequestMismatchError as e: self.fwk.error( 'There has been a fatal error, %s requested too many GPUs per node to launch task %d (requested: ppn = %d, gpp = %d)', caller_id, diff --git a/ipsframework/topicManager.py b/ipsframework/topic_manager.py similarity index 69% rename from ipsframework/topicManager.py rename to ipsframework/topic_manager.py index b7334b1a..4daf9be9 100644 --- a/ipsframework/topicManager.py +++ b/ipsframework/topic_manager.py @@ -12,17 +12,17 @@ processing by any listener are purged periodically as a result of a listener activity like processing or unregistering. For cases where the event list could grow unbounded in the absence of any listener activity for prolonged periods of -time, we could define a 'limitPendingEvents' parameter to denote a bound on the +time, we could define a 'limit_pending_events' parameter to denote a bound on the count of pending events exceeding which triggers an event cleanup to remove events that outlive a 'timeToLive' parameter. """ -from .cca_es_spec import Event, EventServiceException +from .cca_es_spec import Event, EventServiceError from .debug import debug class TopicManager: - def __init__(self, limitPendingEvents=10): + def __init__(self, limit_pending_events=10): """eventList is the common listing of events posted to a topic.""" self.eventList = [] @@ -43,16 +43,16 @@ def __init__(self, limitPendingEvents=10): upper limit on the number of events permitted to be pending for this topic at any point in time. Not pressed into service yet. """ - self.limitPendingEvents = limitPendingEvents + self.limit_pending_events = limit_pending_events debug.output('TopicManager.__init__') - self.printEventsAndListeners() + self.print_events_and_listeners() """ Append the event to the event list. """ - def sendEvent(self, theEvent): + def send_event(self, the_event): """ A new event is appended to the end of the event list provided there is at least one registered listener. This is in accordance with the policy that @@ -63,13 +63,13 @@ def sendEvent(self, theEvent): no registered listeners. """ if len(self.listenerDirectory) > 0: - self.eventList.append(theEvent) - eventList_len = len(self.eventList) - self.maxPendingEvents = max(eventList_len, self.maxPendingEvents) - debug.output('TopicManager.sendEvent') - self.printEventsAndListeners() + self.eventList.append(the_event) + event_list_len = len(self.eventList) + self.maxPendingEvents = max(event_list_len, self.maxPendingEvents) + debug.output('TopicManager.send_event') + self.print_events_and_listeners() - def registerListener(self, listenerid): + def register_listener(self, listenerid): """ For a new listener, the event list marker is initialized to the end of the event list in accordance with the policy that listeners receive only those @@ -77,10 +77,10 @@ def registerListener(self, listenerid): """ if listenerid not in self.listenerDirectory: self.listenerDirectory[listenerid] = len(self.eventList) - debug.output('TopicManager.registerListener') - self.printEventsAndListeners() + debug.output('TopicManager.register_listener') + self.print_events_and_listeners() else: - raise EventServiceException('Event listener registered earlier.') + raise EventServiceError('Event listener registered earlier.') """ A listener activity like unregistering or processing triggers cleanup of @@ -88,58 +88,58 @@ def registerListener(self, listenerid): of all registered listeners. """ - def cleanupEvents(self, listenerid): + def cleanup_events(self, listenerid): self.listenerDirectory[listenerid] = len(self.eventList) """ First determine the oldest pending event. """ - oldestPendingEvent = min(self.listenerDirectory.values()) + oldest_pending_event = min(self.listenerDirectory.values()) """ Reset current listeners' list markers and remove events having no pending listeners. """ - if oldestPendingEvent > 0: - del self.eventList[:oldestPendingEvent] + if oldest_pending_event > 0: + del self.eventList[:oldest_pending_event] for listener_id in self.listenerDirectory: - self.listenerDirectory[listener_id] -= oldestPendingEvent + self.listenerDirectory[listener_id] -= oldest_pending_event """ A listener is unregistered by first performing an event cleanup, followed by deletion of the listener from listenerDirectory. """ - def unregisterListener(self, listenerid): - self.cleanupEvents(listenerid) + def unregister_listener(self, listenerid): + self.cleanup_events(listenerid) del self.listenerDirectory[listenerid] - debug.output('TopicManager.unregisterListener') - self.printEventsAndListeners() + debug.output('TopicManager.unregister_listener') + self.print_events_and_listeners() """ Returns events posted since the last fetch for a listener. """ - def getEventListForListener(self, listenerid): - eventListForListener = [] - for theEvent in self.eventList[self.listenerDirectory[listenerid] :]: - eventListForListener.append(Event(theEvent.header, theEvent.body)) - self.cleanupEvents(listenerid) - debug.output('TopicManager.getEventListForListener') - self.printEventsAndListeners() - return eventListForListener + def get_event_list_for_listener(self, listenerid): + event_list_for_listener = [] + for the_event in self.eventList[self.listenerDirectory[listenerid] :]: + event_list_for_listener.append(Event(the_event.header, the_event.body)) + self.cleanup_events(listenerid) + debug.output('TopicManager.get_event_list_for_listener') + self.print_events_and_listeners() + return event_list_for_listener """ Print out the contents for debugging. """ - def printEventsAndListeners(self): + def print_events_and_listeners(self): string = ':::::::::\n' + 'List of events:' i = 0 for i, e in enumerate(self.eventList): string += '\n' + str(i) + '---' + str(e) string += '\n\n' + 'List of listeners:' debug.output(string) - sortedKeys = sorted(self.listenerDirectory.keys()) - for listenerid in sortedKeys: + sorted_keys = sorted(self.listenerDirectory.keys()) + for listenerid in sorted_keys: string = 'event = ' + str(self.listenerDirectory[listenerid]) debug.output(string, listenerid) debug.output(':::::::::') @@ -149,5 +149,5 @@ def printEventsAndListeners(self): maxPendingEvents. """ - def getEventStats(self): + def get_event_stats(self): return self.maxPendingEvents diff --git a/pyproject.toml b/pyproject.toml index a8b9d01e..6c463304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,12 +120,12 @@ isort = { known-first-party = ['ipsframework'] } #pydocstyle = { convention = 'google' } flake8-quotes = { inline-quotes = 'single', multiline-quotes = 'double' } mccabe = { max-complexity = 45 } # TODO drop this to 20 -#pylint = { max-args = 10, max-branches = 20, max-returns = 10 } +pylint = { max-positional-args = 15, max-args = 10, max-branches = 20, max-returns = 10 } # pyflakes and the relevant pycodestyle rules are already configured extend-select = [ 'C90', # mccabe complexity 'I', # isort - #'N', # pep8-naming + 'N', # pep8-naming #'D', # pydocstyle #'UP', # pyupgrade #'YTT', # flake8-2020 @@ -166,6 +166,38 @@ extend-select = [ ] # If you're seeking to disable a rule, first consider whether the rule is overbearing, or if it should only be turned off for your usecase. ignore = [ + ### TODO move these to extend-select when ready + 'D', # pydocstyle + 'UP', # pyupgrade + 'YTT', # flake8-2020 + 'ANN', # flake8-annotations + 'ASYNC', # flake8-async + 'S', # flake8-bandit + 'BLE', # flake8-blind-except + 'A', # flake8-builtins + 'DTZ', # flake8-datetimez + 'T10', # flake8-debugger + 'EM', # flake8-error-message + 'FA', # flake8-future-annotations + 'ISC', # flake8-implicit-string-concat + 'ICN', # flake8-import-conventions + 'G', # flake8-logging-format + 'INP', # flake8-no-pep420 + 'PIE', # flake8-PIE + 'T20', # flake8-T20 + 'PYI', # flake8-pyi + 'PT', # flake8-pytest-style + 'RSE', # flake8-raise + 'RET', # flake8-return + 'SLF', # flake8-self + 'SLOT', # flake8-slots + 'SIM', # flake8-simplify + 'TCH', # flake8-type-checking + 'ARG', # flake8-unused-arguments + 'PTH', # flake8-use-pathlib + 'PGH', # pygrep-hooks + 'TRY', # tryceratops + ### things we actually want to ignore are below here ### 'COM812', # formatter, handled by Ruff format 'ISC001', # formatter, handled by Ruff format 'SIM105', # "with contextlib.suppress():" is slower than try-except-pass diff --git a/tests/components/drivers/basic_concurrent1.py b/tests/components/drivers/basic_concurrent1.py index 230a8c4a..fdf347c6 100644 --- a/tests/components/drivers/basic_concurrent1.py +++ b/tests/components/drivers/basic_concurrent1.py @@ -10,10 +10,10 @@ """ from ipsframework import Component -from ipsframework.ipsExceptions import IncompleteCallException +from ipsframework.ipsExceptions import IncompleteCallError -class basic_concurrent1(Component): +class BasicConcurrent1(Component): def init(self, timestamp=0.0, **keywords): self.services.log('Initing') @@ -59,7 +59,7 @@ def step(self, timestamp=0.0, **keywords): try: services.wait_call_list([w2_call_id, w3_call_id], block=False) - except IncompleteCallException as e: + except IncompleteCallError as e: print(str(e)) services.wait_call_list([w2_call_id, w3_call_id]) @@ -69,8 +69,8 @@ def step(self, timestamp=0.0, **keywords): services.call(w2, 'finalize', 99) services.call(w3, 'finalize', 99) - def process_event(self, topicName, theEvent): - print('Driver: processed ', (topicName, str(theEvent))) + def process_event(self, topic_name, the_event): + print('Driver: processed ', (topic_name, str(the_event))) def terminate(self, status): self.services.log('Really Calling terminate()') diff --git a/tests/components/drivers/basic_serial2.py b/tests/components/drivers/basic_serial_1.py similarity index 94% rename from tests/components/drivers/basic_serial2.py rename to tests/components/drivers/basic_serial_1.py index c6cdc234..018df52b 100644 --- a/tests/components/drivers/basic_serial2.py +++ b/tests/components/drivers/basic_serial_1.py @@ -12,7 +12,7 @@ from ipsframework import Component -class basic_serial2(Component): +class BasicSerial1(Component): def init(self, timestamp=0.0, **keywords): self.services.log('Initing') @@ -61,8 +61,8 @@ def step(self, timestamp=0.0, **keywords): services.call(w2, 'finalize', 99) services.call(w3, 'finalize', 99) - def process_event(self, topicName, theEvent): - print('Driver: processed ', (topicName, str(theEvent))) + def process_event(self, topic_name, the_event): + print('Driver: processed ', (topic_name, str(the_event))) def terminate(self, status): self.services.log('Really Calling terminate()') diff --git a/tests/components/drivers/basic_serial1.py b/tests/components/drivers/basic_serial_2.py similarity index 94% rename from tests/components/drivers/basic_serial1.py rename to tests/components/drivers/basic_serial_2.py index 1efafbe0..d9bb2223 100644 --- a/tests/components/drivers/basic_serial1.py +++ b/tests/components/drivers/basic_serial_2.py @@ -12,7 +12,7 @@ from ipsframework import Component -class basic_serial1(Component): +class BasicSerial2(Component): def init(self, timestamp=0.0, **keywords): self.services.log('Initing') @@ -61,8 +61,8 @@ def step(self, timestamp=0.0, **keywords): services.call(w2, 'finalize', 99) services.call(w3, 'finalize', 99) - def process_event(self, topicName, theEvent): - print('Driver: processed ', (topicName, str(theEvent))) + def process_event(self, topic_name, the_event): + print('Driver: processed ', (topic_name, str(the_event))) def terminate(self, status): self.services.log('Really Calling terminate()') diff --git a/tests/components/drivers/driver.py b/tests/components/drivers/driver.py index 3e2f28ac..358c4466 100644 --- a/tests/components/drivers/driver.py +++ b/tests/components/drivers/driver.py @@ -1,7 +1,7 @@ from ipsframework import Component -class driver(Component): +class Driver(Component): def step(self, timestamp=0.0, **keywords): ports = self.services.get_config_param('PORTS') port_names = ports['NAMES'].split() diff --git a/tests/components/drivers/driver_dataManager.py b/tests/components/drivers/driver_dataManager.py index da43e2ef..0f064b76 100644 --- a/tests/components/drivers/driver_dataManager.py +++ b/tests/components/drivers/driver_dataManager.py @@ -1,7 +1,7 @@ from ipsframework import Component -class driver_dataManager(Component): +class DriverDataManager(Component): def step(self, timestamp=0.0, **keywords): self.services.stage_state() @@ -18,4 +18,6 @@ def step(self, timestamp=0.0, **keywords): self.services.update_state() - self.services.merge_current_state('partial_state_file', logfile='merge_current_state.log', merge_binary='echo') + self.services.merge_current_state( + 'partial_state_file', logfile='merge_current_state.log', merge_binary='echo' + ) diff --git a/tests/components/drivers/driver_double_trace.py b/tests/components/drivers/driver_double_trace.py index 4c13a4e1..09507560 100644 --- a/tests/components/drivers/driver_double_trace.py +++ b/tests/components/drivers/driver_double_trace.py @@ -1,7 +1,7 @@ from ipsframework import Component -class driver(Component): +class Driver(Component): def step(self, timestamp=0.0, **keywords): w = self.services.get_port('WORKER') # call the same worker step twice to check that the trace is correct diff --git a/tests/components/drivers/init_dataManager.py b/tests/components/drivers/init_data_manager.py similarity index 92% rename from tests/components/drivers/init_dataManager.py rename to tests/components/drivers/init_data_manager.py index dbeb51e2..80cc30bc 100644 --- a/tests/components/drivers/init_dataManager.py +++ b/tests/components/drivers/init_data_manager.py @@ -1,7 +1,7 @@ from ipsframework import Component -class init_dataManager(Component): +class InitDataManager(Component): def step(self, timestamp=0.0, **keywords): state_file_list = self.services.get_config_param('STATE_FILES').split(' ') diff --git a/tests/components/drivers/logging_tester.py b/tests/components/drivers/logging_tester.py index 93920d66..742840ba 100644 --- a/tests/components/drivers/logging_tester.py +++ b/tests/components/drivers/logging_tester.py @@ -1,9 +1,17 @@ from ipsframework import Component -log_types = ['log', 'debug', 'info', 'warning', 'error', 'exception', 'critical'] +log_types = [ + 'log', + 'debug', + 'info', + 'warning', + 'error', + 'exception', + 'critical', +] -class logging_tester(Component): +class LoggingTester(Component): def init(self, timestamp=0.0, **keywords): print(f'{self.component_id}.init') for log_type in log_types: @@ -14,7 +22,9 @@ def step(self, timestamp=0.0, **keywords): for log_type in log_types: getattr(self.services, log_type)(f'step msg: {log_type}') # with string formatting arguments - getattr(self.services, log_type)(f'step msg: {log_type} timestamp=%d %s', timestamp, 'test') + getattr(self.services, log_type)( + f'step msg: {log_type} timestamp=%d %s', timestamp, 'test' + ) def finalize(self, timestamp=0.0, **keywords): print(f'{self.component_id}.finalize') diff --git a/tests/components/drivers/simple_driver.py b/tests/components/drivers/simple_driver.py index f59e3fde..6b24e02c 100644 --- a/tests/components/drivers/simple_driver.py +++ b/tests/components/drivers/simple_driver.py @@ -1,7 +1,7 @@ from ipsframework import Component -class driver(Component): +class Driver(Component): def step(self, timestamp=0.0, **keywords): w = self.services.get_port('WORKER') self.services.call(w, 'step', 0) diff --git a/tests/components/drivers/timeloop_driver.py b/tests/components/drivers/timeloop_driver.py index 14bc85e5..2a6bf33a 100644 --- a/tests/components/drivers/timeloop_driver.py +++ b/tests/components/drivers/timeloop_driver.py @@ -1,7 +1,7 @@ from ipsframework import Component -class timeloop_driver(Component): +class TimeloopDriver(Component): def init(self, timestamp=0.0, **keywords): self.state_file = self.services.get_config_param('CURRENT_STATE') # pylint: disable=attribute-defined-outside-init self.workers = [ @@ -10,7 +10,11 @@ def init(self, timestamp=0.0, **keywords): if port not in ('INIT', 'DRIVER') ] - mode = 'restart' if self.services.get_config_param('SIMULATION_MODE').lower() == 'restart' else 'init' + mode = ( + 'restart' + if self.services.get_config_param('SIMULATION_MODE').lower() == 'restart' + else 'init' + ) if mode == 'init': with open(self.state_file, 'w') as f: diff --git a/tests/components/workers/bad_workers.py b/tests/components/workers/bad_workers.py index c2725b45..76357962 100644 --- a/tests/components/workers/bad_workers.py +++ b/tests/components/workers/bad_workers.py @@ -5,14 +5,14 @@ def func(x): return x + 1 -class bad_task_worker(Component): +class BadTaskWorker(Component): def step(self, timestamp=0.0, **keywords): cwd = self.services.get_working_dir() pid = self.services.launch_task(1, cwd, 42) self.services.wait_task(pid) -class bad_task_pool_worker1(Component): +class BadTaskPoolWorker1(Component): def step(self, timestamp=0.0, **keywords): cwd = self.services.get_working_dir() self.services.create_task_pool('pool') @@ -21,7 +21,7 @@ def step(self, timestamp=0.0, **keywords): self.services.get_finished_tasks('pool') -class bad_task_pool_worker2(Component): +class BadTaskPoolWorker2(Component): def step(self, timestamp=0.0, **keywords): cwd = self.services.get_working_dir() self.services.create_task_pool('pool') @@ -30,11 +30,11 @@ def step(self, timestamp=0.0, **keywords): self.services.get_finished_tasks('pool') -class exception_worker(Component): +class ExceptionWorker(Component): def step(self, timestamp=0.0, **keywords): raise RuntimeError('Runtime error') -class assign_protected_attribute(Component): +class AssignProtectedAttribute(Component): def step(self, timestamp=0.0, **keywords): self.args = 0 diff --git a/tests/components/workers/cori_srun_openmp.py b/tests/components/workers/cori_srun_openmp.py index 4032d36f..e1e0d185 100644 --- a/tests/components/workers/cori_srun_openmp.py +++ b/tests/components/workers/cori_srun_openmp.py @@ -1,7 +1,7 @@ from ipsframework import Component -class openmp_task(Component): +class OpenmpTask(Component): # pylint: disable=no-member def step(self, timestamp=0.0, **keywords): cwd = self.services.get_working_dir() @@ -9,28 +9,82 @@ def step(self, timestamp=0.0, **keywords): mpi = '/usr/common/software/bin/check-mpi.gnu.cori' hybrid = '/usr/common/software/bin/check-hybrid.gnu.cori' - self.services.wait_task(self.services.launch_task(1, cwd, mpi, logfile='log.01', errfile='err.01', omp=True)) - self.services.wait_task(self.services.launch_task(1, cwd, mpi, logfile='log.02', errfile='err.02', task_ppn=1, omp=True)) - self.services.wait_task(self.services.launch_task(1, cwd, mpi, logfile='log.03', errfile='err.03', task_ppn=1, task_cpp=32, omp=True)) + self.services.wait_task( + self.services.launch_task(1, cwd, mpi, logfile='log.01', errfile='err.01', omp=True) + ) + self.services.wait_task( + self.services.launch_task( + 1, cwd, mpi, logfile='log.02', errfile='err.02', task_ppn=1, omp=True + ) + ) + self.services.wait_task( + self.services.launch_task( + 1, cwd, mpi, logfile='log.03', errfile='err.03', task_ppn=1, task_cpp=32, omp=True + ) + ) - self.services.wait_task(self.services.launch_task(4, cwd, mpi, logfile='log.11', errfile='err.11', omp=True)) - self.services.wait_task(self.services.launch_task(4, cwd, mpi, logfile='log.12', errfile='err.12', task_ppn=4, omp=True)) - self.services.wait_task(self.services.launch_task(4, cwd, mpi, logfile='log.13', errfile='err.13', task_ppn=4, task_cpp=8, omp=True)) + self.services.wait_task( + self.services.launch_task(4, cwd, mpi, logfile='log.11', errfile='err.11', omp=True) + ) + self.services.wait_task( + self.services.launch_task( + 4, cwd, mpi, logfile='log.12', errfile='err.12', task_ppn=4, omp=True + ) + ) + self.services.wait_task( + self.services.launch_task( + 4, cwd, mpi, logfile='log.13', errfile='err.13', task_ppn=4, task_cpp=8, omp=True + ) + ) - self.services.wait_task(self.services.launch_task(32, cwd, mpi, logfile='log.21', errfile='err.21', omp=True)) - self.services.wait_task(self.services.launch_task(32, cwd, mpi, logfile='log.22', errfile='err.22', task_ppn=32, omp=True)) - self.services.wait_task(self.services.launch_task(32, cwd, mpi, logfile='log.23', errfile='err.23', task_ppn=32, task_cpp=1, omp=True)) + self.services.wait_task( + self.services.launch_task(32, cwd, mpi, logfile='log.21', errfile='err.21', omp=True) + ) + self.services.wait_task( + self.services.launch_task( + 32, cwd, mpi, logfile='log.22', errfile='err.22', task_ppn=32, omp=True + ) + ) + self.services.wait_task( + self.services.launch_task( + 32, cwd, mpi, logfile='log.23', errfile='err.23', task_ppn=32, task_cpp=1, omp=True + ) + ) - self.services.wait_task(self.services.launch_task(4, cwd, mpi, logfile='log.31', errfile='err.31', task_ppn=8, omp=True)) - self.services.wait_task(self.services.launch_task(4, cwd, mpi, logfile='log.32', errfile='err.32', task_ppn=4, task_cpp=4, omp=True)) - self.services.wait_task(self.services.launch_task(4, cwd, mpi, logfile='log.33', errfile='err.33', task_ppn=4, task_cpp=2, omp=True)) + self.services.wait_task( + self.services.launch_task( + 4, cwd, mpi, logfile='log.31', errfile='err.31', task_ppn=8, omp=True + ) + ) + self.services.wait_task( + self.services.launch_task( + 4, cwd, mpi, logfile='log.32', errfile='err.32', task_ppn=4, task_cpp=4, omp=True + ) + ) + self.services.wait_task( + self.services.launch_task( + 4, cwd, mpi, logfile='log.33', errfile='err.33', task_ppn=4, task_cpp=2, omp=True + ) + ) - self.services.wait_task(self.services.launch_task(4, cwd, hybrid, logfile='log.41', errfile='err.41', task_ppn=8, omp=True)) - self.services.wait_task(self.services.launch_task(4, cwd, hybrid, logfile='log.42', errfile='err.42', task_ppn=4, task_cpp=4, omp=True)) - self.services.wait_task(self.services.launch_task(4, cwd, hybrid, logfile='log.43', errfile='err.43', task_ppn=4, task_cpp=2, omp=True)) + self.services.wait_task( + self.services.launch_task( + 4, cwd, hybrid, logfile='log.41', errfile='err.41', task_ppn=8, omp=True + ) + ) + self.services.wait_task( + self.services.launch_task( + 4, cwd, hybrid, logfile='log.42', errfile='err.42', task_ppn=4, task_cpp=4, omp=True + ) + ) + self.services.wait_task( + self.services.launch_task( + 4, cwd, hybrid, logfile='log.43', errfile='err.43', task_ppn=4, task_cpp=2, omp=True + ) + ) -class openmp_task_pool(Component): +class OpenmpTaskPool(Component): # pylint: disable=no-member def step(self, timestamp=0.0, **keywords): cwd = self.services.get_working_dir() @@ -38,8 +92,32 @@ def step(self, timestamp=0.0, **keywords): self.services.create_task_pool('pool') mpi = '/usr/common/software/bin/check-mpi.gnu.cori' - self.services.add_task('pool', 'task_1', 4, cwd, mpi, logfile='log.1', errfile='err.1', task_ppn=8, omp=True) - self.services.add_task('pool', 'task_2', 4, cwd, mpi, logfile='log.2', errfile='err.2', task_ppn=4, task_cpp=4, omp=True) - self.services.add_task('pool', 'task_3', 4, cwd, mpi, logfile='log.3', errfile='err.3', task_ppn=4, task_cpp=2, omp=True) + self.services.add_task( + 'pool', 'task_1', 4, cwd, mpi, logfile='log.1', errfile='err.1', task_ppn=8, omp=True + ) + self.services.add_task( + 'pool', + 'task_2', + 4, + cwd, + mpi, + logfile='log.2', + errfile='err.2', + task_ppn=4, + task_cpp=4, + omp=True, + ) + self.services.add_task( + 'pool', + 'task_3', + 4, + cwd, + mpi, + logfile='log.3', + errfile='err.3', + task_ppn=4, + task_cpp=2, + omp=True, + ) self.services.submit_tasks('pool') diff --git a/tests/components/workers/dask_worker.py b/tests/components/workers/dask_worker.py index 5580edd1..b5dfeabe 100644 --- a/tests/components/workers/dask_worker.py +++ b/tests/components/workers/dask_worker.py @@ -1,7 +1,7 @@ from ipsframework import Component -class dask_worker(Component): +class DaskWorker(Component): # pylint: disable=no-member def step(self, timestamp=0.0, **keywords): cmd = self.EXECUTABLE @@ -20,13 +20,25 @@ def step(self, timestamp=0.0, **keywords): if self.ERRFILE: kwargs['errfile'] = self.ERRFILE.format(i) - self.services.add_task('pool', f'task_{i}', int(self.NPROC), cwd, cmd, self.VALUE if self.VALUE else f'{i}', **kwargs) + self.services.add_task( + 'pool', + f'task_{i}', + int(self.NPROC), + cwd, + cmd, + self.VALUE if self.VALUE else f'{i}', + **kwargs, + ) nodes = self.services.get_config_param('NODES') ret_val = self.services.submit_tasks( - 'pool', use_dask=True, use_shifter=self.SHIFTER == 'True', dask_nodes=nodes, dask_worker_per_gpu=self.GPU == 'True' + 'pool', + use_dask=True, + use_shifter=self.SHIFTER == 'True', + dask_nodes=nodes, + dask_worker_per_gpu=self.GPU == 'True', ) self.services.info('ret_val = %d', ret_val) exit_status = self.services.get_finished_tasks('pool') for i in range(total_tasks): task_name = f'task_{i}' - self.services.info('{} {}'.format(task_name, exit_status.get(task_name))) + self.services.info(f'{task_name} {exit_status.get(task_name)}') diff --git a/tests/components/workers/large_worker.py b/tests/components/workers/large_worker.py index a0eda713..b57fdd93 100644 --- a/tests/components/workers/large_worker.py +++ b/tests/components/workers/large_worker.py @@ -6,7 +6,7 @@ from ipsframework import Component -class large_worker(Component): +class LargeWorker(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) @@ -22,15 +22,21 @@ def step(self, timestamp=0.0, **keywords): sleep_time = 1 self.services.log('Stepping Worker timestamp=%s', timestamp) cwd = self.services.get_working_dir() - pid = self.services.launch_task(int(self.NPROC), cwd, os.path.join(self.BIN_PATH, self.BIN), str(sleep_time), logfile='my_out' + timestamp) + pid = self.services.launch_task( + int(self.NPROC), + cwd, + os.path.join(self.BIN_PATH, self.BIN), + str(sleep_time), + logfile='my_out' + timestamp, + ) retval = self.services.wait_task(pid) return retval def finalize(self, timestamp=0.0, **keywords): self.services.log('Finalizing Worker') - def process_event(self, topicName, theEvent): - print('Worker: processed ', (topicName, str(theEvent))) + def process_event(self, topic_name, the_event): + print('Worker: processed ', (topic_name, str(the_event))) def terminate(self, status): self.services.log('Really Calling terminate()') diff --git a/tests/components/workers/medium_worker.py b/tests/components/workers/medium_worker.py index d1f9473f..7cfb03b6 100644 --- a/tests/components/workers/medium_worker.py +++ b/tests/components/workers/medium_worker.py @@ -6,7 +6,7 @@ from ipsframework import Component -class medium_worker(Component): +class MediumWorker(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) @@ -22,15 +22,21 @@ def step(self, timestamp=0.0, **keywords): sleep_time = 1 self.services.log('Stepping Worker timestamp=%s', timestamp) cwd = self.services.get_working_dir() - pid = self.services.launch_task(int(self.NPROC), cwd, os.path.join(self.BIN_PATH, self.BIN), str(sleep_time), logfile='my_out' + timestamp) + pid = self.services.launch_task( + int(self.NPROC), + cwd, + os.path.join(self.BIN_PATH, self.BIN), + str(sleep_time), + logfile='my_out' + timestamp, + ) retval = self.services.wait_task(pid) return retval def finalize(self, timestamp=0.0, **keywords): self.services.log('Finalizing Worker') - def process_event(self, topicName, theEvent): - print('Worker: processed ', (topicName, str(theEvent))) + def process_event(self, topic_name, the_event): + print('Worker: processed ', (topic_name, str(the_event))) def terminate(self, status): self.services.log('Really Calling terminate()') diff --git a/tests/components/workers/perlmutter_srun_gpu.py b/tests/components/workers/perlmutter_srun_gpu.py index 5dee41f0..af2d4fc6 100644 --- a/tests/components/workers/perlmutter_srun_gpu.py +++ b/tests/components/workers/perlmutter_srun_gpu.py @@ -1,7 +1,7 @@ from ipsframework import Component -class gpu_task(Component): +class GpuTask(Component): # pylint: disable=no-member def step(self, timestamp=0.0, **keywords): cwd = self.services.get_working_dir() diff --git a/tests/components/workers/simple_sleep.py b/tests/components/workers/simple_sleep.py index a424d383..a4cb10be 100644 --- a/tests/components/workers/simple_sleep.py +++ b/tests/components/workers/simple_sleep.py @@ -6,7 +6,7 @@ from ipsframework import Component -class simple_sleep(Component): +class SimpleSleep(Component): def step(self, timestamp=0.0, **keywords): time.sleep(1) self.services.wait_task(self.services.launch_task(1, '/tmp', '/bin/sleep', 1)) diff --git a/tests/components/workers/small_worker.py b/tests/components/workers/small_worker.py index 23432fc8..38527659 100644 --- a/tests/components/workers/small_worker.py +++ b/tests/components/workers/small_worker.py @@ -6,7 +6,7 @@ from ipsframework import Component -class small_worker(Component): +class SmallWorker(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) @@ -22,15 +22,21 @@ def step(self, timestamp=0.0, **keywords): sleep_time = 1 self.services.log('Stepping Worker timestamp=%s', timestamp) cwd = self.services.get_working_dir() - pid = self.services.launch_task(int(self.NPROC), cwd, os.path.join(self.BIN_PATH, self.BIN), str(sleep_time), logfile='my_out' + timestamp) + pid = self.services.launch_task( + int(self.NPROC), + cwd, + os.path.join(self.BIN_PATH, self.BIN), + str(sleep_time), + logfile='my_out' + timestamp, + ) retval = self.services.wait_task(pid) return retval def finalize(self, timestamp=0.0, **keywords): self.services.log('Finalizing Worker') - def process_event(self, topicName, theEvent): - print('Worker: processed ', (topicName, str(theEvent))) + def process_event(self, topic_name, the_event): + print('Worker: processed ', (topic_name, str(the_event))) def terminate(self, status): self.services.log('Really Calling terminate()') diff --git a/tests/components/workers/timeloop_comp.py b/tests/components/workers/timeloop_comp.py index 2deb3921..0914f186 100644 --- a/tests/components/workers/timeloop_comp.py +++ b/tests/components/workers/timeloop_comp.py @@ -2,7 +2,7 @@ # pylint: disable=no-member,attribute-defined-outside-init -class timeloop_comp(Component): +class TimeloopComp(Component): def init(self, timestamp=0.0, **keywords): self.output_files = self.OUTPUT_FILES.split() self.output_files.append(self.services.get_config_param('CURRENT_STATE')) diff --git a/tests/conftest.py b/tests/conftest.py index 06514898..67dc053c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import pytest -from ipsframework.componentRegistry import ComponentID +from ipsframework.component_registry import ComponentID try: from pytest_cov.embed import cleanup_on_sigterm @@ -23,7 +23,7 @@ def run_around_tests(): else: def on_terminate(proc): - print('Process {} terminated with exit code {}'.format(proc, proc.returncode)) + print(f'Process {proc} terminated with exit code {proc.returncode}') @pytest.fixture(autouse=True) def run_around_tests(): @@ -48,7 +48,9 @@ def run_around_tests(): def pytest_addoption(parser): parser.addoption('--runcori', action='store_true', default=False, help='run Cori tests') - parser.addoption('--runperlmutter', action='store_true', default=False, help='run Perlmutter tests') + parser.addoption( + '--runperlmutter', action='store_true', default=False, help='run Perlmutter tests' + ) def pytest_configure(config): diff --git a/tests/dakota/dakota_test_Gaussian.py b/tests/dakota/dakota_test_Gaussian.py old mode 100755 new mode 100644 diff --git a/tests/dakota/dakota_test_Rosenbrock.ips b/tests/dakota/dakota_test_Rosenbrock.ips index a241cc36..9666ad48 100644 --- a/tests/dakota/dakota_test_Rosenbrock.ips +++ b/tests/dakota/dakota_test_Rosenbrock.ips @@ -15,29 +15,29 @@ RUN_COMMENT = Testing dakota # Specification of plasma state files -# Where to put plasma state files as the simulation evolves +# Where to put plasma state files as the simulation evolves PLASMA_STATE_WORK_DIR = $SIM_ROOT/work/plasma_state # Specify what files constitute the plasma state - N.B. not all components need all files -PLASMA_STATE_FILES = +PLASMA_STATE_FILES = # Names of ports to be used. An implementation and configuration must be specified for # each port [PORTS] - NAMES = DRIVER - + NAMES = DRIVER + # DRIVER port is called by the framework. It is required, causes exception. - [[DRIVER]] # REQUIRED Port section + [[DRIVER]] # REQUIRED Port section IMPLEMENTATION = ROSE - + # INIT port is called by the framework. It typically produces the very first set of # plasma state files for SIMULATION_MODE = NORMAL. It does not raise and exception # if missing. - - [[INIT]] - IMPLEMENTATION = + + [[INIT]] + IMPLEMENTATION = # Specification of IMPLEMENTATION for each physics port called out in PORTS list. # Additional specifications may be present that are not in the PORTS list @@ -49,12 +49,12 @@ PLASMA_STATE_FILES = [ROSE] CLASS = DAKOTA SUB_CLASS = TEST - NAME = ResenbrockDriver + NAME = RosenbrockDriver NPROC = 1 BIN_PATH = INPUT_DIR = - INPUT_FILES = - OUTPUT_FILES = + INPUT_FILES = + OUTPUT_FILES = SCRIPT = $PWD/dakota_test_Rosenbrock.py # Time loop specification (two modes for now) EXPLICIT | REGULAR @@ -63,6 +63,6 @@ PLASMA_STATE_FILES = [TIME_LOOP] MODE = REGULAR - START = 0 - FINISH = 10 + START = 0 + FINISH = 10 NSTEP = 10 diff --git a/tests/dakota/dakota_test_Rosenbrock.py b/tests/dakota/dakota_test_Rosenbrock.py old mode 100755 new mode 100644 index f0bcc810..605052cf --- a/tests/dakota/dakota_test_Rosenbrock.py +++ b/tests/dakota/dakota_test_Rosenbrock.py @@ -6,7 +6,7 @@ from ipsframework import Component -class ResenbrockDriver(Component): +class RosenbrockDriver(Component): def init(self, timestamp=0.0, **keywords): print('init from dakota test driver') diff --git a/tests/dakota/test_dakota.py b/tests/dakota/test_dakota.py index 676c2c0b..7e289aef 100644 --- a/tests/dakota/test_dakota.py +++ b/tests/dakota/test_dakota.py @@ -11,22 +11,25 @@ def copy_config_and_replace(infile, outfile, tmpdir): - with open(infile, 'r') as fin: - with open(outfile, 'w') as fout: - for line in fin: - if 'SCRIPT' in line: - fout.write(line.replace('$PWD', str(tmpdir))) - elif line.startswith('SIM_ROOT'): - fout.write(f'SIM_ROOT = {tmpdir}/$SIM_NAME\n') - else: - fout.write(line) + with open(infile, 'r') as fin, open(outfile, 'w') as fout: + for line in fin: + if 'SCRIPT' in line: + fout.write(line.replace('$PWD', str(tmpdir))) + elif line.startswith('SIM_ROOT'): + fout.write(f'SIM_ROOT = {tmpdir}/$SIM_NAME\n') + else: + fout.write(line) @pytest.mark.skipif(shutil.which('dakota') is None, reason='Requires dakota to run this test') @pytest.mark.timeout(200) def test_dakota(tmpdir): data_dir = os.path.dirname(__file__) - copy_config_and_replace(os.path.join(data_dir, 'dakota_test_Gaussian.ips'), tmpdir.join('dakota_test_Gaussian.ips'), tmpdir) + copy_config_and_replace( + os.path.join(data_dir, 'dakota_test_Gaussian.ips'), + tmpdir.join('dakota_test_Gaussian.ips'), + tmpdir, + ) shutil.copy(os.path.join(data_dir, 'workstation.conf'), tmpdir) shutil.copy(os.path.join(data_dir, 'dakota_test_Gaussian.in'), tmpdir) shutil.copy(os.path.join(data_dir, 'dakota_test_Gaussian.py'), tmpdir) @@ -48,13 +51,15 @@ def test_dakota(tmpdir): with open(log_file, 'r') as f: lines = f.readlines() - X = lines[-13].split()[1] + x = lines[-13].split()[1] - assert float(X) == pytest.approx(0.5, rel=1e-4) + assert float(x) == pytest.approx(0.5, rel=1e-4) # Check PARENT CHILD relationship # Get parent PORTAL_RUNID - json_files = glob.glob(str(tmpdir.join('DAKOTA_Gaussian_TEST_1').join('simulation_log').join('*.json'))) + json_files = glob.glob( + str(tmpdir.join('DAKOTA_Gaussian_TEST_1').join('simulation_log').join('*.json')) + ) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: @@ -71,7 +76,14 @@ def test_dakota(tmpdir): assert user == 'user' # Check child run - json_files = glob.glob(str(tmpdir.join('DAKOTA_Gaussian_TEST_1').join('simulation_*_0000').join('simulation_log').join('*.json'))) + json_files = glob.glob( + str( + tmpdir.join('DAKOTA_Gaussian_TEST_1') + .join('simulation_*_0000') + .join('simulation_log') + .join('*.json') + ) + ) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: lines = json_file.readlines() @@ -92,43 +104,43 @@ def test_dakota(tmpdir): @mock.patch('ipsframework.ips_dakota_dynamic.DakotaDynamic') -def test_dakota_main(MockDakotaDynamic): +def test_dakota_main(mock_dakota_dynamic): # override sys.argv for testing sys.argv = ['ips_dakota_dynamic.py'] ret = ips_dakota_dynamic.main() assert ret == 1 - MockDakotaDynamic.assert_not_called() + mock_dakota_dynamic.assert_not_called() - MockDakotaDynamic.reset_mock() + mock_dakota_dynamic.reset_mock() ret = ips_dakota_dynamic.main(['ips_dakota_dynamic.py']) assert ret == 1 - MockDakotaDynamic.assert_not_called() + mock_dakota_dynamic.assert_not_called() - MockDakotaDynamic.reset_mock() + mock_dakota_dynamic.reset_mock() sys.argv = ['ips_dakota_dynamic.py', '--somethingelse'] ret = ips_dakota_dynamic.main() assert ret == 1 - MockDakotaDynamic.assert_not_called() + mock_dakota_dynamic.assert_not_called() - MockDakotaDynamic.reset_mock() + mock_dakota_dynamic.reset_mock() sys.argv = ['ips_dakota_dynamic.py', '--dakotaconfig=dakota.cfg'] ret = ips_dakota_dynamic.main() assert ret == 1 - MockDakotaDynamic.assert_not_called() + mock_dakota_dynamic.assert_not_called() - MockDakotaDynamic.reset_mock() + mock_dakota_dynamic.reset_mock() sys.argv = ['ips_dakota_dynamic.py', '--simulation=sim.cfg'] ret = ips_dakota_dynamic.main() assert ret == 1 - MockDakotaDynamic.assert_not_called() + mock_dakota_dynamic.assert_not_called() - MockDakotaDynamic.reset_mock() + mock_dakota_dynamic.reset_mock() sys.argv = ['ips_dakota_dynamic.py', '--simulation=sim.cfg', '--dakotaconfig=dakota.cfg'] ret = ips_dakota_dynamic.main() assert ret == 0 - MockDakotaDynamic.assert_called_with('dakota.cfg', None, None, False, 'sim.cfg', None) + mock_dakota_dynamic.assert_called_with('dakota.cfg', None, None, False, 'sim.cfg', None) - MockDakotaDynamic.reset_mock() + mock_dakota_dynamic.reset_mock() sys.argv = [ 'ips_dakota_dynamic.py', '--simulation=sim.cfg', @@ -140,4 +152,6 @@ def test_dakota_main(MockDakotaDynamic): ] ret = ips_dakota_dynamic.main() assert ret == 0 - MockDakotaDynamic.assert_called_with('dakota.cfg', 'out.log', 'computer.conf', True, 'sim.cfg', 'dakota.rst') + mock_dakota_dynamic.assert_called_with( + 'dakota.cfg', 'out.log', 'computer.conf', True, 'sim.cfg', 'dakota.rst' + ) diff --git a/tests/ensembles/already-exists/a_sim_comp.py b/tests/ensembles/already-exists/a_sim_comp.py index cdd2d782..85fe2c95 100644 --- a/tests/ensembles/already-exists/a_sim_comp.py +++ b/tests/ensembles/already-exists/a_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `a_sim`.""" from ipsframework import Component -class a_sim_comp(Component): +class ASimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/already-exists/another_sim_comp.py b/tests/ensembles/already-exists/another_sim_comp.py index f1d12a02..4cfdd7bf 100644 --- a/tests/ensembles/already-exists/another_sim_comp.py +++ b/tests/ensembles/already-exists/another_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `another_sim`.""" from ipsframework import Component -class another_sim_comp(Component): +class AnotherSimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/already-exists/driver.config b/tests/ensembles/already-exists/driver.config index b45752a0..547aed93 100644 --- a/tests/ensembles/already-exists/driver.config +++ b/tests/ensembles/already-exists/driver.config @@ -19,7 +19,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/already-exists/ CLASS = driver SUB_CLASS = - NAME = ensemble_driver + NAME = EnsembleDriver SCRIPT = ${BIN_PATH}/driver.py TEMPLATE = ${BIN_PATH}/template.config PLATFORM_CONFIG_FILE = ${BIN_PATH}/mac135909.config @@ -27,5 +27,3 @@ SIMULATION_MODE = NORMAL INPUT_FILES = OUTPUT_FILES = RESTART_FILES = - - diff --git a/tests/ensembles/already-exists/driver.py b/tests/ensembles/already-exists/driver.py index 0e432aad..fb2ea2c2 100644 --- a/tests/ensembles/already-exists/driver.py +++ b/tests/ensembles/already-exists/driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -11,12 +10,12 @@ from ipsframework import Component -class ensemble_driver(Component): +class EnsembleDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): @@ -64,7 +63,11 @@ def step(self, timestamp=0.0, **keywords): # for variable substitutions. variables = { 'a_sim_comp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], 'C': ['bar', 'baz', 'quux']}, - 'another_sim_comp': {'D': [7, 5, 9], 'B': [0.775, 0.080, 29.2], 'F': ['xyzzy', 'plud', 'thud']}, + 'another_sim_comp': { + 'D': [7, 5, 9], + 'B': [0.775, 0.080, 29.2], + 'F': ['xyzzy', 'plud', 'thud'], + }, } # Spins up N tasks, in this case three, each with a different set of @@ -75,5 +78,5 @@ def step(self, timestamp=0.0, **keywords): self.services.info(f'Mapping of dirs to parameters: {mapping!s}') - def finalize(self, timeStamp=0.0): + def finalize(self, time_stamp=0.0): return diff --git a/tests/ensembles/already-exists/instance_driver.py b/tests/ensembles/already-exists/instance_driver.py index 142333a7..3b0278e5 100644 --- a/tests/ensembles/already-exists/instance_driver.py +++ b/tests/ensembles/already-exists/instance_driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -9,12 +8,12 @@ from ipsframework import Component -class instance_driver(Component): +class InstanceDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating instance driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): diff --git a/tests/ensembles/already-exists/template.config b/tests/ensembles/already-exists/template.config index 5a256ee4..4b2cf362 100644 --- a/tests/ensembles/already-exists/template.config +++ b/tests/ensembles/already-exists/template.config @@ -23,7 +23,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/already-exists/ CLASS = driver SUB_CLASS = - NAME = instance_driver + NAME = InstanceDriver SCRIPT = ${BIN_PATH}/instance_driver.py NPROC = 1 INPUT_FILES = @@ -35,7 +35,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/already-exists/ CLASS = workers SUB_CLASS = - NAME = a_sim_comp + NAME = ASimComp SCRIPT = ${BIN_PATH}/a_sim_comp.py NPROC = 1 INPUT_FILES = @@ -50,7 +50,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/already-exists/ CLASS = workers SUB_CLASS = - NAME = another_sim_comp + NAME = AnotherSimComp SCRIPT = ${BIN_PATH}/another_sim_comp.py NPROC = 1 INPUT_FILES = diff --git a/tests/ensembles/basic/a_sim_comp.py b/tests/ensembles/basic/a_sim_comp.py index cdd2d782..85fe2c95 100644 --- a/tests/ensembles/basic/a_sim_comp.py +++ b/tests/ensembles/basic/a_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `a_sim`.""" from ipsframework import Component -class a_sim_comp(Component): +class ASimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/basic/another_sim_comp.py b/tests/ensembles/basic/another_sim_comp.py index f1d12a02..4cfdd7bf 100644 --- a/tests/ensembles/basic/another_sim_comp.py +++ b/tests/ensembles/basic/another_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `another_sim`.""" from ipsframework import Component -class another_sim_comp(Component): +class AnotherSimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/basic/driver.config b/tests/ensembles/basic/driver.config index aff5fe10..fa077ba4 100644 --- a/tests/ensembles/basic/driver.config +++ b/tests/ensembles/basic/driver.config @@ -19,7 +19,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/basic/ CLASS = driver SUB_CLASS = - NAME = ensemble_driver + NAME = EnsembleDriver SCRIPT = ${BIN_PATH}/driver.py TEMPLATE = ${BIN_PATH}/template.config PLATFORM_CONFIG_FILE = ${BIN_PATH}/mac135909.config @@ -27,5 +27,3 @@ SIMULATION_MODE = NORMAL INPUT_FILES = OUTPUT_FILES = RESTART_FILES = - - diff --git a/tests/ensembles/basic/driver.py b/tests/ensembles/basic/driver.py index 0e432aad..fb2ea2c2 100644 --- a/tests/ensembles/basic/driver.py +++ b/tests/ensembles/basic/driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -11,12 +10,12 @@ from ipsframework import Component -class ensemble_driver(Component): +class EnsembleDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): @@ -64,7 +63,11 @@ def step(self, timestamp=0.0, **keywords): # for variable substitutions. variables = { 'a_sim_comp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], 'C': ['bar', 'baz', 'quux']}, - 'another_sim_comp': {'D': [7, 5, 9], 'B': [0.775, 0.080, 29.2], 'F': ['xyzzy', 'plud', 'thud']}, + 'another_sim_comp': { + 'D': [7, 5, 9], + 'B': [0.775, 0.080, 29.2], + 'F': ['xyzzy', 'plud', 'thud'], + }, } # Spins up N tasks, in this case three, each with a different set of @@ -75,5 +78,5 @@ def step(self, timestamp=0.0, **keywords): self.services.info(f'Mapping of dirs to parameters: {mapping!s}') - def finalize(self, timeStamp=0.0): + def finalize(self, time_stamp=0.0): return diff --git a/tests/ensembles/basic/instance_driver.py b/tests/ensembles/basic/instance_driver.py index 142333a7..3b0278e5 100644 --- a/tests/ensembles/basic/instance_driver.py +++ b/tests/ensembles/basic/instance_driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -9,12 +8,12 @@ from ipsframework import Component -class instance_driver(Component): +class InstanceDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating instance driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): diff --git a/tests/ensembles/basic/template.config b/tests/ensembles/basic/template.config index a5f35d6f..19a86b22 100644 --- a/tests/ensembles/basic/template.config +++ b/tests/ensembles/basic/template.config @@ -23,7 +23,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/basic/ CLASS = driver SUB_CLASS = - NAME = instance_driver + NAME = InstanceDriver SCRIPT = ${BIN_PATH}/instance_driver.py NPROC = 1 INPUT_FILES = @@ -35,7 +35,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/basic/ CLASS = workers SUB_CLASS = - NAME = a_sim_comp + NAME = ASimComp SCRIPT = ${BIN_PATH}/a_sim_comp.py NPROC = 1 INPUT_FILES = @@ -50,7 +50,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/ CLASS = workers SUB_CLASS = - NAME = another_sim_comp + NAME = AnotherSimComp SCRIPT = ${BIN_PATH}/another_sim_comp.py NPROC = 1 INPUT_FILES = diff --git a/tests/ensembles/dask/a_sim_comp.py b/tests/ensembles/dask/a_sim_comp.py index cdd2d782..85fe2c95 100644 --- a/tests/ensembles/dask/a_sim_comp.py +++ b/tests/ensembles/dask/a_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `a_sim`.""" from ipsframework import Component -class a_sim_comp(Component): +class ASimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/dask/another_sim_comp.py b/tests/ensembles/dask/another_sim_comp.py index f1d12a02..4cfdd7bf 100644 --- a/tests/ensembles/dask/another_sim_comp.py +++ b/tests/ensembles/dask/another_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `another_sim`.""" from ipsframework import Component -class another_sim_comp(Component): +class AnotherSimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/dask/driver.config b/tests/ensembles/dask/driver.config index 084be29a..858be19d 100644 --- a/tests/ensembles/dask/driver.config +++ b/tests/ensembles/dask/driver.config @@ -19,12 +19,10 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/dask/ CLASS = driver SUB_CLASS = - NAME = ensemble_driver + NAME = EnsembleDriver SCRIPT = ${BIN_PATH}/driver.py TEMPLATE = ${BIN_PATH}/template.config NPROC = 1 INPUT_FILES = OUTPUT_FILES = RESTART_FILES = - - diff --git a/tests/ensembles/dask/driver.py b/tests/ensembles/dask/driver.py index 2f77686e..6193b51b 100644 --- a/tests/ensembles/dask/driver.py +++ b/tests/ensembles/dask/driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -11,12 +10,12 @@ from ipsframework import Component -class ensemble_driver(Component): +class EnsembleDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): @@ -58,7 +57,11 @@ def step(self, timestamp=0.0, **keywords): # for variable substitutions. variables = { 'a_sim_comp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], 'C': ['bar', 'baz', 'quux']}, - 'another_sim_comp': {'D': [7, 5, 9], 'B': [0.775, 0.080, 29.2], 'F': ['xyzzy', 'plud', 'thud']}, + 'another_sim_comp': { + 'D': [7, 5, 9], + 'B': [0.775, 0.080, 29.2], + 'F': ['xyzzy', 'plud', 'thud'], + }, } # Spins up N tasks, in this case three, each with a different set of @@ -74,5 +77,5 @@ def step(self, timestamp=0.0, **keywords): self.services.info(f'Mapping of dirs to parameters: {mapping!s}') - def finalize(self, timeStamp=0.0): + def finalize(self, time_stamp=0.0): return diff --git a/tests/ensembles/dask/instance_driver.py b/tests/ensembles/dask/instance_driver.py index 142333a7..3b0278e5 100644 --- a/tests/ensembles/dask/instance_driver.py +++ b/tests/ensembles/dask/instance_driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -9,12 +8,12 @@ from ipsframework import Component -class instance_driver(Component): +class InstanceDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating instance driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): diff --git a/tests/ensembles/dask/template.config b/tests/ensembles/dask/template.config index b26d2f3d..9158ca26 100644 --- a/tests/ensembles/dask/template.config +++ b/tests/ensembles/dask/template.config @@ -23,7 +23,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/dask/ CLASS = driver SUB_CLASS = - NAME = instance_driver + NAME = InstanceDriver SCRIPT = ${BIN_PATH}/instance_driver.py NPROC = 1 INPUT_FILES = @@ -35,7 +35,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/dask/ CLASS = workers SUB_CLASS = - NAME = a_sim_comp + NAME = ASimComp SCRIPT = ${BIN_PATH}/a_sim_comp.py NPROC = 1 INPUT_FILES = @@ -50,7 +50,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/ CLASS = workers SUB_CLASS = - NAME = another_sim_comp + NAME = AnotherSimComp SCRIPT = ${BIN_PATH}/another_sim_comp.py NPROC = 1 INPUT_FILES = diff --git a/tests/ensembles/extra-variable/a_sim_comp.py b/tests/ensembles/extra-variable/a_sim_comp.py index cdd2d782..85fe2c95 100644 --- a/tests/ensembles/extra-variable/a_sim_comp.py +++ b/tests/ensembles/extra-variable/a_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `a_sim`.""" from ipsframework import Component -class a_sim_comp(Component): +class ASimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/extra-variable/another_sim_comp.py b/tests/ensembles/extra-variable/another_sim_comp.py index f1d12a02..4cfdd7bf 100644 --- a/tests/ensembles/extra-variable/another_sim_comp.py +++ b/tests/ensembles/extra-variable/another_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `another_sim`.""" from ipsframework import Component -class another_sim_comp(Component): +class AnotherSimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/extra-variable/driver.config b/tests/ensembles/extra-variable/driver.config index 1bc48b6e..60bb3cad 100644 --- a/tests/ensembles/extra-variable/driver.config +++ b/tests/ensembles/extra-variable/driver.config @@ -19,7 +19,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/extra-variable/ CLASS = driver SUB_CLASS = - NAME = ensemble_driver + NAME = EnsembleDriver SCRIPT = ${BIN_PATH}/driver.py TEMPLATE = ${BIN_PATH}/template.config PLATFORM_CONFIG_FILE = ${BIN_PATH}/mac135909.config @@ -27,5 +27,3 @@ SIMULATION_MODE = NORMAL INPUT_FILES = OUTPUT_FILES = RESTART_FILES = - - diff --git a/tests/ensembles/extra-variable/driver.py b/tests/ensembles/extra-variable/driver.py index 0e432aad..fb2ea2c2 100644 --- a/tests/ensembles/extra-variable/driver.py +++ b/tests/ensembles/extra-variable/driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -11,12 +10,12 @@ from ipsframework import Component -class ensemble_driver(Component): +class EnsembleDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): @@ -64,7 +63,11 @@ def step(self, timestamp=0.0, **keywords): # for variable substitutions. variables = { 'a_sim_comp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], 'C': ['bar', 'baz', 'quux']}, - 'another_sim_comp': {'D': [7, 5, 9], 'B': [0.775, 0.080, 29.2], 'F': ['xyzzy', 'plud', 'thud']}, + 'another_sim_comp': { + 'D': [7, 5, 9], + 'B': [0.775, 0.080, 29.2], + 'F': ['xyzzy', 'plud', 'thud'], + }, } # Spins up N tasks, in this case three, each with a different set of @@ -75,5 +78,5 @@ def step(self, timestamp=0.0, **keywords): self.services.info(f'Mapping of dirs to parameters: {mapping!s}') - def finalize(self, timeStamp=0.0): + def finalize(self, time_stamp=0.0): return diff --git a/tests/ensembles/extra-variable/instance_driver.py b/tests/ensembles/extra-variable/instance_driver.py index 142333a7..3b0278e5 100644 --- a/tests/ensembles/extra-variable/instance_driver.py +++ b/tests/ensembles/extra-variable/instance_driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -9,12 +8,12 @@ from ipsframework import Component -class instance_driver(Component): +class InstanceDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating instance driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): diff --git a/tests/ensembles/extra-variable/template.config b/tests/ensembles/extra-variable/template.config index 536b2ac9..c10c684d 100644 --- a/tests/ensembles/extra-variable/template.config +++ b/tests/ensembles/extra-variable/template.config @@ -23,7 +23,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/extra-variable/ CLASS = driver SUB_CLASS = - NAME = instance_driver + NAME = InstanceDriver SCRIPT = ${BIN_PATH}/instance_driver.py NPROC = 1 INPUT_FILES = @@ -35,7 +35,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/extra-variable/ CLASS = workers SUB_CLASS = - NAME = a_sim_comp + NAME = ASimComp SCRIPT = ${BIN_PATH}/a_sim_comp.py NPROC = 1 INPUT_FILES = @@ -50,7 +50,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/ CLASS = workers SUB_CLASS = - NAME = another_sim_comp + NAME = AnotherSimComp SCRIPT = ${BIN_PATH}/another_sim_comp.py NPROC = 1 INPUT_FILES = diff --git a/tests/ensembles/missing-assignment/a_sim_comp.py b/tests/ensembles/missing-assignment/a_sim_comp.py index cdd2d782..85fe2c95 100644 --- a/tests/ensembles/missing-assignment/a_sim_comp.py +++ b/tests/ensembles/missing-assignment/a_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `a_sim`.""" from ipsframework import Component -class a_sim_comp(Component): +class ASimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/missing-assignment/another_sim_comp.py b/tests/ensembles/missing-assignment/another_sim_comp.py index f1d12a02..4cfdd7bf 100644 --- a/tests/ensembles/missing-assignment/another_sim_comp.py +++ b/tests/ensembles/missing-assignment/another_sim_comp.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 """Component wrapper for the ensemble example for `another_sim`.""" from ipsframework import Component -class another_sim_comp(Component): +class AnotherSimComp(Component): def __init__(self, services, config): super().__init__(services, config) print('Created %s' % (self.__class__)) diff --git a/tests/ensembles/missing-assignment/driver.config b/tests/ensembles/missing-assignment/driver.config index f155a60d..18967c10 100644 --- a/tests/ensembles/missing-assignment/driver.config +++ b/tests/ensembles/missing-assignment/driver.config @@ -19,7 +19,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/missing-assignment/ CLASS = driver SUB_CLASS = - NAME = ensemble_driver + NAME = EnsembleDriver SCRIPT = ${BIN_PATH}/driver.py TEMPLATE = ${BIN_PATH}/template.config PLATFORM_CONFIG_FILE = ${BIN_PATH}/mac135909.config @@ -27,5 +27,3 @@ SIMULATION_MODE = NORMAL INPUT_FILES = OUTPUT_FILES = RESTART_FILES = - - diff --git a/tests/ensembles/missing-assignment/driver.py b/tests/ensembles/missing-assignment/driver.py index 0e432aad..fb2ea2c2 100644 --- a/tests/ensembles/missing-assignment/driver.py +++ b/tests/ensembles/missing-assignment/driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -11,12 +10,12 @@ from ipsframework import Component -class ensemble_driver(Component): +class EnsembleDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): @@ -64,7 +63,11 @@ def step(self, timestamp=0.0, **keywords): # for variable substitutions. variables = { 'a_sim_comp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], 'C': ['bar', 'baz', 'quux']}, - 'another_sim_comp': {'D': [7, 5, 9], 'B': [0.775, 0.080, 29.2], 'F': ['xyzzy', 'plud', 'thud']}, + 'another_sim_comp': { + 'D': [7, 5, 9], + 'B': [0.775, 0.080, 29.2], + 'F': ['xyzzy', 'plud', 'thud'], + }, } # Spins up N tasks, in this case three, each with a different set of @@ -75,5 +78,5 @@ def step(self, timestamp=0.0, **keywords): self.services.info(f'Mapping of dirs to parameters: {mapping!s}') - def finalize(self, timeStamp=0.0): + def finalize(self, time_stamp=0.0): return diff --git a/tests/ensembles/missing-assignment/instance_driver.py b/tests/ensembles/missing-assignment/instance_driver.py index 142333a7..3b0278e5 100644 --- a/tests/ensembles/missing-assignment/instance_driver.py +++ b/tests/ensembles/missing-assignment/instance_driver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Example driver for the ensembles example. @@ -9,12 +8,12 @@ from ipsframework import Component -class instance_driver(Component): +class InstanceDriver(Component): def __init__(self, services, config): super().__init__(services, config) print('Creating instance driver') - def init(self, timeStamp=0.0): + def init(self, time_stamp=0.0): return def step(self, timestamp=0.0, **keywords): diff --git a/tests/ensembles/missing-assignment/template.config b/tests/ensembles/missing-assignment/template.config index a870bb51..47c718aa 100644 --- a/tests/ensembles/missing-assignment/template.config +++ b/tests/ensembles/missing-assignment/template.config @@ -23,7 +23,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/missing-assignment/ CLASS = driver SUB_CLASS = - NAME = instance_driver + NAME = InstanceDriver SCRIPT = ${BIN_PATH}/instance_driver.py NPROC = 1 INPUT_FILES = @@ -35,7 +35,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/missing-assignment/ CLASS = workers SUB_CLASS = - NAME = a_sim_comp + NAME = ASimComp SCRIPT = ${BIN_PATH}/a_sim_comp.py NPROC = 1 INPUT_FILES = @@ -51,7 +51,7 @@ SIMULATION_MODE = NORMAL BIN_PATH = $SRC_DIR/IPS-framework/tests/ensembles/missing-assignment/ CLASS = workers SUB_CLASS = - NAME = another_sim_comp + NAME = AnotherSimComp SCRIPT = ${BIN_PATH}/another_sim_comp.py NPROC = 1 INPUT_FILES = diff --git a/tests/hello-world-nested/test_hello-world-nested.py b/tests/hello-world-nested/test_hello-world-nested.py index 76f50809..84c9ff0e 100644 --- a/tests/hello-world-nested/test_hello-world-nested.py +++ b/tests/hello-world-nested/test_hello-world-nested.py @@ -7,21 +7,26 @@ def copy_config_and_replace(infile, outfile, tmpdir): - with open(infile, 'r') as fin: - with open(outfile, 'w') as fout: - for line in fin: - if line.startswith('TEST_ROOT'): - fout.write(f'TEST_ROOT = {tmpdir}\n') - elif line.startswith('LOG_FILE'): - fout.write(line.replace('LOG_FILE = ', f'LOG_FILE = {tmpdir}/')) - else: - fout.write(line) + with open(infile, 'r') as fin, open(outfile, 'w') as fout: + for line in fin: + if line.startswith('TEST_ROOT'): + fout.write(f'TEST_ROOT = {tmpdir}\n') + elif line.startswith('LOG_FILE'): + fout.write(line.replace('LOG_FILE = ', f'LOG_FILE = {tmpdir}/')) + else: + fout.write(line) def test_hello_world_nested(tmpdir, capfd): data_dir = os.path.dirname(__file__) - copy_config_and_replace(os.path.join(data_dir, 'hello_world.config'), tmpdir.join('hello_world.config'), tmpdir) - copy_config_and_replace(os.path.join(data_dir, 'hello_world_sub.config'), tmpdir.join('hello_world_sub.config'), tmpdir) + copy_config_and_replace( + os.path.join(data_dir, 'hello_world.config'), tmpdir.join('hello_world.config'), tmpdir + ) + copy_config_and_replace( + os.path.join(data_dir, 'hello_world_sub.config'), + tmpdir.join('hello_world_sub.config'), + tmpdir, + ) shutil.copy(os.path.join(data_dir, 'workstation.conf'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_driver.py'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_worker.py'), tmpdir) @@ -65,7 +70,9 @@ def test_hello_world_nested(tmpdir, capfd): assert 'WORKERSSUB_HELLO_HelloWorker_6 INFO Hello from HelloWorker - sub\n' in lines # check sub workflow results file - sub_out = tmpdir.join('hello_example_SUPER/work/WORKERS_HELLO_HelloWorker_2/Subflow_01/simulation_results/DRIVERS_HELLOSUB_HelloDriver_5/sub_out_0.0.txt') + sub_out = tmpdir.join( + 'hello_example_SUPER/work/WORKERS_HELLO_HelloWorker_2/Subflow_01/simulation_results/DRIVERS_HELLOSUB_HelloDriver_5/sub_out_0.0.txt' + ) assert os.path.exists(str(sub_out)) assert os.path.islink(str(sub_out)) @@ -95,7 +102,9 @@ def test_hello_world_nested(tmpdir, capfd): assert lines[0] == 'SUB INPUT FILE\n' - sub_input = tmpdir.join('hello_example_SUPER/work/WORKERS_HELLO_HelloWorker_2/HELLO_DRIVER/input.txt') + sub_input = tmpdir.join( + 'hello_example_SUPER/work/WORKERS_HELLO_HelloWorker_2/HELLO_DRIVER/input.txt' + ) assert os.path.exists(str(sub_input)) @@ -105,10 +114,12 @@ def test_hello_world_nested(tmpdir, capfd): assert lines[0] == 'SUB INPUT FILE\n' # check the simulation log json - json_files = glob.glob(str(tmpdir.join('hello_example_SUPER').join('simulation_log').join('*.json'))) + json_files = glob.glob( + str(tmpdir.join('hello_example_SUPER').join('simulation_log').join('*.json')) + ) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: - events = [json.loads(event) for event in json_file.readlines()] + events = [json.loads(event) for event in json_file] assert len(events) == 25 assert events[-1]['eventtype'] == 'IPS_END' diff --git a/tests/helloworld/hello_worker_task_pool_dask.py b/tests/helloworld/hello_worker_task_pool_dask.py index b1e6924a..ad1f1446 100644 --- a/tests/helloworld/hello_worker_task_pool_dask.py +++ b/tests/helloworld/hello_worker_task_pool_dask.py @@ -9,8 +9,8 @@ from ipsframework import Component -def myFun(*args): - print(f'myFun({args[0]})') +def my_fun(*args): + print(f'my_fun({args[0]})') sleep(float(args[0])) return 0 @@ -36,17 +36,21 @@ def step(self, timestamp=0.0, **keywords): self.services.create_task_pool('pool') for i, duration in enumerate(('0.2', '0.4', '0.6')): self.services.add_task('pool', 'bin_' + str(i), 1, cwd, exe, duration) - self.services.add_task('pool', 'meth_' + str(i), 1, cwd, copy.copy(self).myMethod, duration) - self.services.add_task('pool', 'func_' + str(i), 1, cwd, myFun, duration) + self.services.add_task( + 'pool', 'meth_' + str(i), 1, cwd, copy.copy(self).my_method, duration + ) + self.services.add_task('pool', 'func_' + str(i), 1, cwd, my_fun, duration) worker_plugin = DaskWorkerPlugin() - ret_val = self.services.submit_tasks('pool', use_dask=True, dask_nodes=1, dask_ppw=10, dask_worker_plugin=worker_plugin) + ret_val = self.services.submit_tasks( + 'pool', use_dask=True, dask_nodes=1, dask_ppw=10, dask_worker_plugin=worker_plugin + ) print('ret_val = ', ret_val) exit_status = self.services.get_finished_tasks('pool') print(exit_status) - def myMethod(self, *args): - print(f'myMethod({args[0]})') + def my_method(self, *args): + print(f'my_method({args[0]})') sleep(float(args[0])) return 0 diff --git a/tests/helloworld/test_helloworld.py b/tests/helloworld/test_helloworld.py index b62e5b4d..9566a7e5 100644 --- a/tests/helloworld/test_helloworld.py +++ b/tests/helloworld/test_helloworld.py @@ -6,31 +6,34 @@ def copy_config_and_replace(infile, outfile, tmpdir, worker='hello_worker.py', portal=False): - with open(infile, 'r') as fin: - with open(outfile, 'w') as fout: - for line in fin: - if 'hello_driver.py' in line or 'hello_worker.py' in line: - fout.write(line.replace('${BIN_PATH}', str(tmpdir)).replace('hello_worker.py', worker)) - elif 'BIN_PATH' in line: - fout.write(line.replace('${IPS_ROOT}/tests/helloworld', '')) - elif line.startswith('SIM_ROOT'): - fout.write(f'SIM_ROOT = {tmpdir}\n') - elif line.startswith('LOG_FILE'): - fout.write(line.replace('LOG_FILE = ', f'LOG_FILE = {tmpdir}/')) - elif line.startswith('USE_PORTAL'): - if portal: - fout.write('USE_PORTAL = True\n') - fout.write(f'USER_W3_DIR = {tmpdir}/www\n') - fout.write('PORTAL_URL = http://localhost:8080\n') - else: - fout.write(line) + with open(infile, 'r') as fin, open(outfile, 'w') as fout: + for line in fin: + if 'hello_driver.py' in line or 'hello_worker.py' in line: + fout.write( + line.replace('${BIN_PATH}', str(tmpdir)).replace('hello_worker.py', worker) + ) + elif 'BIN_PATH' in line: + fout.write(line.replace('${IPS_ROOT}/tests/helloworld', '')) + elif line.startswith('SIM_ROOT'): + fout.write(f'SIM_ROOT = {tmpdir}\n') + elif line.startswith('LOG_FILE'): + fout.write(line.replace('LOG_FILE = ', f'LOG_FILE = {tmpdir}/')) + elif line.startswith('USE_PORTAL'): + if portal: + fout.write('USE_PORTAL = True\n') + fout.write(f'USER_W3_DIR = {tmpdir}/www\n') + fout.write('PORTAL_URL = http://localhost:8080\n') else: fout.write(line) + else: + fout.write(line) def test_helloworld(tmpdir, capfd): data_dir = os.path.dirname(__file__) - copy_config_and_replace(os.path.join(data_dir, 'hello_world.ips'), tmpdir.join('hello_world.ips'), tmpdir) + copy_config_and_replace( + os.path.join(data_dir, 'hello_world.ips'), tmpdir.join('hello_world.ips'), tmpdir + ) shutil.copy(os.path.join(data_dir, 'platform.conf'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_driver.py'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_worker.py'), tmpdir) @@ -49,7 +52,7 @@ def test_helloworld(tmpdir, capfd): fwk_components = framework.config_manager.get_framework_components() assert len(fwk_components) == 1 - assert 'Hello_world_1_FWK@runspaceInitComponent@3' in fwk_components + assert 'Hello_world_1_FWK@RunspaceInitComponent@3' in fwk_components component_map = framework.config_manager.get_component_map() @@ -85,7 +88,12 @@ def test_helloworld(tmpdir, capfd): def test_helloworld_launch_task(tmpdir, capfd): data_dir = os.path.dirname(__file__) - copy_config_and_replace(os.path.join(data_dir, 'hello_world.ips'), tmpdir.join('hello_world.ips'), tmpdir, worker='hello_worker_launch_task.py') + copy_config_and_replace( + os.path.join(data_dir, 'hello_world.ips'), + tmpdir.join('hello_world.ips'), + tmpdir, + worker='hello_worker_launch_task.py', + ) shutil.copy(os.path.join(data_dir, 'platform.conf'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_driver.py'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_worker_launch_task.py'), tmpdir) @@ -104,7 +112,7 @@ def test_helloworld_launch_task(tmpdir, capfd): fwk_components = framework.config_manager.get_framework_components() assert len(fwk_components) == 1 - assert 'Hello_world_1_FWK@runspaceInitComponent@3' in fwk_components + assert 'Hello_world_1_FWK@RunspaceInitComponent@3' in fwk_components component_map = framework.config_manager.get_component_map() @@ -149,7 +157,12 @@ def test_helloworld_launch_task(tmpdir, capfd): def test_helloworld_task_pool(tmpdir, capfd): data_dir = os.path.dirname(__file__) - copy_config_and_replace(os.path.join(data_dir, 'hello_world.ips'), tmpdir.join('hello_world.ips'), tmpdir, worker='hello_worker_task_pool.py') + copy_config_and_replace( + os.path.join(data_dir, 'hello_world.ips'), + tmpdir.join('hello_world.ips'), + tmpdir, + worker='hello_worker_task_pool.py', + ) shutil.copy(os.path.join(data_dir, 'platform.conf'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_driver.py'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_worker_task_pool.py'), tmpdir) @@ -220,7 +233,12 @@ def test_helloworld_task_pool_dask(tmpdir, capfd): assert TaskPool.dask is not None data_dir = os.path.dirname(__file__) - copy_config_and_replace(os.path.join(data_dir, 'hello_world.ips'), tmpdir.join('hello_world.ips'), tmpdir, worker='hello_worker_task_pool_dask.py') + copy_config_and_replace( + os.path.join(data_dir, 'hello_world.ips'), + tmpdir.join('hello_world.ips'), + tmpdir, + worker='hello_worker_task_pool_dask.py', + ) shutil.copy(os.path.join(data_dir, 'platform.conf'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_driver.py'), tmpdir) shutil.copy(os.path.join(data_dir, 'hello_worker_task_pool_dask.py'), tmpdir) @@ -255,7 +273,7 @@ def test_helloworld_task_pool_dask(tmpdir, capfd): assert 'ret_val = 9' in captured_out for duration in ('0.2', '0.4', '0.6'): - for task in ['myFun', 'myMethod']: + for task in ['my_fun', 'myMethod']: assert f'{task}({duration})' in captured_out exit_status = json.loads(captured_out[-3].replace("'", '"')) diff --git a/tests/multirun/basic_concurrent1.ips b/tests/multirun/basic_concurrent_1.ips similarity index 91% rename from tests/multirun/basic_concurrent1.ips rename to tests/multirun/basic_concurrent_1.ips index 523f2ca0..78b090df 100644 --- a/tests/multirun/basic_concurrent1.ips +++ b/tests/multirun/basic_concurrent_1.ips @@ -3,7 +3,7 @@ OUTPUT_PREFIX = RUN_COMMENT = testing basic concurrent component simulation capability RUN_ID = test # Identifier for this simulation run -TOKAMAK_ID = basic_concurrent1 # Identifier for tokamak simulated +TOKAMAK_ID = basic_concurrent_1 # Identifier for tokamak simulated SHOT_NUMBER = 0 # Identifier for specific case for this tokamak (not necessarily a number) SIM_NAME = ${RUN_ID}_${TOKAMAK_ID}_${SHOT_NUMBER} # Name of current simulation @@ -41,17 +41,17 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [BASIC_CONCURRENT1] CLASS = drivers SUB_CLASS = testing - NAME = basic_concurrent1 + NAME = BasicConcurrent1 NPROC = 1 BIN_PATH = $IPS_ROOT/tests/components/drivers INPUT_FILES = file1 sfile1 sfile2 ofile1 ofile2 OUTPUT_FILES = ofile1 ofile2 - SCRIPT = $BIN_PATH/basic_concurrent1.py + SCRIPT = $BIN_PATH/basic_concurrent_1.py [SMALL_WORKER] CLASS = workers SUB_CLASS = testing - NAME = small_worker + NAME = SmallWorker NPROC = 1 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep @@ -62,7 +62,7 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [MEDIUM_WORKER] CLASS = workers SUB_CLASS = testing - NAME = medium_worker + NAME = MediumWorker NPROC = 1 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep @@ -73,7 +73,7 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [LARGE_WORKER] CLASS = workers SUB_CLASS = testing - NAME = large_worker + NAME = LargeWorker NPROC = 2 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep diff --git a/tests/multirun/basic_serial1.ips b/tests/multirun/basic_serial_1.ips similarity index 87% rename from tests/multirun/basic_serial1.ips rename to tests/multirun/basic_serial_1.ips index 8af9ff37..574ce78c 100644 --- a/tests/multirun/basic_serial1.ips +++ b/tests/multirun/basic_serial_1.ips @@ -1,9 +1,9 @@ -OUTPUT_PREFIX = +OUTPUT_PREFIX = RUN_COMMENT = testing basic concurrent component simulation capability RUN_ID = test # Identifier for this simulation run -TOKAMAK_ID = basic_serial1 # Identifier for tokamak simulated +TOKAMAK_ID = basic_serial_1 # Identifier for tokamak simulated SHOT_NUMBER = 0 # Identifier for specific case for this tokamak (not necessarily a number) SIM_NAME = ${RUN_ID}_${TOKAMAK_ID}_${SHOT_NUMBER} # Name of current simulation @@ -12,10 +12,10 @@ SIM_ROOT = $PWD/$SIM_NAME # Where to put results from this SIMULATION_MODE = NORMAL STATE_WORK_DIR = $SIM_ROOT/work/plasma_state # Where to put plasma state files as the simulation evolves -CURRENT_STATE = -PRIOR_STATE = -NEXT_STATE = -CURRENT_EQDSK = +CURRENT_STATE = +PRIOR_STATE = +NEXT_STATE = +CURRENT_EQDSK = USE_PORTAL=True LOG_FILE = $SIM_ROOT/$SIM_NAME.log @@ -23,8 +23,8 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [PORTS] NAMES = DRIVER WORKER1 WORKER2 WORKER3 - - [[DRIVER]] # REQUIRED Port section + + [[DRIVER]] # REQUIRED Port section IMPLEMENTATION = BASIC_SERIAL1 [[WORKER1]] @@ -41,17 +41,17 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [BASIC_SERIAL1] CLASS = drivers SUB_CLASS = testing - NAME = basic_serial1 + NAME = BasicSerial1 NPROC = 1 BIN_PATH = $IPS_ROOT/tests/components/drivers INPUT_FILES = file1 sfile1 sfile2 ofile1 ofile2 - OUTPUT_FILES = ofile1 ofile2 - SCRIPT = $BIN_PATH/basic_serial1.py + OUTPUT_FILES = ofile1 ofile2 + SCRIPT = $BIN_PATH/basic_serial_1.py [SMALL_WORKER] CLASS = workers SUB_CLASS = testing - NAME = small_worker + NAME = SmallWorker NPROC = 1 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep @@ -62,7 +62,7 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [MEDIUM_WORKER] CLASS = workers SUB_CLASS = testing - NAME = medium_worker + NAME = MediumWorker NPROC = 1 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep @@ -73,7 +73,7 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [LARGE_WORKER] CLASS = workers SUB_CLASS = testing - NAME = large_worker + NAME = LargeWorker NPROC = 2 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep diff --git a/tests/multirun/basic_serial2.ips b/tests/multirun/basic_serial_2.ips similarity index 87% rename from tests/multirun/basic_serial2.ips rename to tests/multirun/basic_serial_2.ips index f1bb797d..1adee350 100644 --- a/tests/multirun/basic_serial2.ips +++ b/tests/multirun/basic_serial_2.ips @@ -1,9 +1,9 @@ -OUTPUT_PREFIX = +OUTPUT_PREFIX = RUN_COMMENT = testing basic concurrent component simulation capability RUN_ID = test # Identifier for this simulation run -TOKAMAK_ID = basic_serial2 # Identifier for tokamak simulated +TOKAMAK_ID = basic_serial_2 # Identifier for tokamak simulated SHOT_NUMBER = 0 # Identifier for specific case for this tokamak (not necessarily a number) SIM_NAME = ${RUN_ID}_${TOKAMAK_ID}_${SHOT_NUMBER} # Name of current simulation @@ -12,10 +12,10 @@ SIM_ROOT = $PWD/$SIM_NAME # Where to put results from this SIMULATION_MODE = NORMAL STATE_WORK_DIR = $SIM_ROOT/work/plasma_state # Where to put plasma state files as the simulation evolves -CURRENT_STATE = -PRIOR_STATE = -NEXT_STATE = -CURRENT_EQDSK = +CURRENT_STATE = +PRIOR_STATE = +NEXT_STATE = +CURRENT_EQDSK = USE_PORTAL=True LOG_FILE = $SIM_ROOT/$SIM_NAME.log @@ -23,8 +23,8 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [PORTS] NAMES = DRIVER WORKER1 WORKER2 WORKER3 - - [[DRIVER]] # REQUIRED Port section + + [[DRIVER]] # REQUIRED Port section IMPLEMENTATION = BASIC_SERIAL2 [[WORKER1]] @@ -41,17 +41,17 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [BASIC_SERIAL2] CLASS = drivers SUB_CLASS = testing - NAME = basic_serial2 + NAME = BasicSerial2 NPROC = 1 BIN_PATH = $IPS_ROOT/tests/components/drivers INPUT_FILES = file1 sfile1 sfile2 ofile1 ofile2 - OUTPUT_FILES = ofile1 ofile2 - SCRIPT = $BIN_PATH/basic_serial2.py - + OUTPUT_FILES = ofile1 ofile2 + SCRIPT = $BIN_PATH/basic_serial_2.py + [SMALL_WORKER] CLASS = workers SUB_CLASS = testing - NAME = small_worker + NAME = SmallWorker NPROC = 1 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep @@ -62,7 +62,7 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [MEDIUM_WORKER] CLASS = workers SUB_CLASS = testing - NAME = medium_worker + NAME = MediumWorker NPROC = 1 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep @@ -73,7 +73,7 @@ LOG_LEVEL = INFO # Possible values: DEBUG, INFO, WARNING, ERRO [LARGE_WORKER] CLASS = workers SUB_CLASS = testing - NAME = large_worker + NAME = LargeWorker NPROC = 2 BIN_PATH = $IPS_ROOT/tests/bin BIN = parallel_sleep diff --git a/tests/multirun/test_basic_serial.py b/tests/multirun/test_basic_serial.py index bf367793..3c8614d7 100644 --- a/tests/multirun/test_basic_serial.py +++ b/tests/multirun/test_basic_serial.py @@ -14,27 +14,27 @@ def copy_config_and_replace(infile, srcdir, tmpdir): for line in fin: if line.startswith('SIM_ROOT'): fout.write(f'SIM_ROOT = {tmpdir}/$SIM_NAME\n') - IPS_ROOT = os.path.abspath(os.path.join(srcdir, '..', '..')) - fout.write(f'IPS_ROOT = {IPS_ROOT}\n') + ips_root = os.path.abspath(os.path.join(srcdir, '..', '..')) + fout.write(f'IPS_ROOT = {ips_root}\n') else: fout.write(line) @pytest.mark.skipif(not shutil.which('mpirun'), reason='requires mpirun') -def test_basic_serial1(tmpdir, capfd): +def test_basic_serial_1(tmpdir, capfd): datadir = os.path.dirname(__file__) - copy_config_and_replace('basic_serial1.ips', datadir, tmpdir) + copy_config_and_replace('basic_serial_1.ips', datadir, tmpdir) shutil.copy(os.path.join(datadir, 'platform.conf'), tmpdir) # setup 'input' files os.system(f'cd {tmpdir}; touch file1 ofile1 ofile2 sfile1 sfile2') framework = Framework( - config_file_list=[os.path.join(tmpdir, 'basic_serial1.ips')], + config_file_list=[os.path.join(tmpdir, 'basic_serial_1.ips')], log_file_name=os.path.join(tmpdir, 'test.log'), platform_file_name=os.path.join(tmpdir, 'platform.conf'), - debug=None, - verbose_debug=None, + debug=False, + verbose_debug=False, cmd_nodes=0, cmd_ppn=0, ) @@ -47,9 +47,9 @@ def test_basic_serial1(tmpdir, capfd): captured_err = captured.err.split('\n') assert captured_err[0].startswith('Starting IPS') - assert captured_out[0] == "Created " - assert captured_out[1] == "Created " - assert captured_out[2] == "Created " + assert captured_out[0] == "Created " + assert captured_out[1] == "Created " + assert captured_out[2] == "Created " assert captured_out[3] == 'small_worker : init() called' assert captured_out[5] == 'medium_worker : init() called' assert captured_out[7] == 'large_worker : init() called' @@ -58,13 +58,33 @@ def test_basic_serial1(tmpdir, capfd): assert captured_out[11] == 'Current time = 3.70' # check files copied and created - driver_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_serial1_0/work/drivers_testing_basic_serial1_*/*')))] + driver_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join('test_basic_serial_1_0/work/drivers_testing_basic_serial_1_*/*')) + ) + ] for infile in ['file1', 'ofile1', 'ofile2', 'sfile1', 'sfile2']: assert infile in driver_files - small_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_serial1_0/work/workers_testing_small_worker_*/*')))] - medium_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_serial1_0/work/workers_testing_medium_worker_*/*')))] - large_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_serial1_0/work/workers_testing_large_worker_*/*')))] + small_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join('test_basic_serial_1_0/work/workers_testing_SmallWorker_*/*')) + ) + ] + medium_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join('test_basic_serial_1_0/work/workers_testing_MediumWorker_*/*')) + ) + ] + large_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join('test_basic_serial_1_0/work/workers_testing_LargeWorker_*/*')) + ) + ] for outfile in ['my_out3.50', 'my_out3.60', 'my_out3.70']: assert outfile in small_worker_files @@ -73,45 +93,65 @@ def test_basic_serial1(tmpdir, capfd): # check contents of my_out files for outfile in ['my_out3.50', 'my_out3.60', 'my_out3.70']: - for worker in ['workers_testing_small_worker_2', 'workers_testing_medium_worker_3']: - with open(str(tmpdir.join('test_basic_serial1_0/work').join(worker).join(outfile)), 'r') as f: + for worker in [ + 'workers_testing_SmallWorker_2', + 'workers_testing_MediumWorker_3', + ]: + with open( + str(tmpdir.join('test_basic_serial_1_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() assert "results = ['Rank 0 slept for 1.0 seconds']\n" in lines - worker = 'workers_testing_large_worker_4' - with open(str(tmpdir.join('test_basic_serial1_0/work').join(worker).join(outfile)), 'r') as f: + worker = 'workers_testing_LargeWorker_4' + with open( + str(tmpdir.join('test_basic_serial_1_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() - assert "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines + assert ( + "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines + ) # check sim log file - with open(str(tmpdir.join('test_basic_serial1_0').join('test_basic_serial1_0.log')), 'r') as f: + with open( + str(tmpdir.join('test_basic_serial_1_0').join('test_basic_serial_1_0.log')), + 'r', + ) as f: lines = f.readlines() # remove timestamp lines = [line[24:] for line in lines] - for worker in ['small_worker_2', 'medium_worker_3', 'large_worker_4']: + for worker in ['SmallWorker_2', 'MediumWorker_3', 'LargeWorker_4']: for timestamp in ['3.50', '3.60', '3.70']: - assert f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' in lines + assert ( + f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' + in lines + ) @pytest.mark.skipif(not shutil.which('mpirun'), reason='requires mpirun') def test_basic_serial_multi(tmpdir, capfd): - # This is the same as test_basic_serial1 except that 2 simulation files are use at the same time + # This is the same as test_basic_serial_1 except that 2 simulation files are use at the same time datadir = os.path.dirname(__file__) - copy_config_and_replace('basic_serial1.ips', datadir, tmpdir) - copy_config_and_replace('basic_serial2.ips', datadir, tmpdir) + copy_config_and_replace('basic_serial_1.ips', datadir, tmpdir) + copy_config_and_replace('basic_serial_2.ips', datadir, tmpdir) shutil.copy(os.path.join(datadir, 'platform.conf'), tmpdir) # setup 'input' files os.system(f'cd {tmpdir}; touch file1 ofile1 ofile2 sfile1 sfile2') framework = Framework( - config_file_list=[os.path.join(tmpdir, 'basic_serial1.ips'), os.path.join(tmpdir, 'basic_serial2.ips')], + config_file_list=[ + os.path.join(tmpdir, 'basic_serial_1.ips'), + os.path.join(tmpdir, 'basic_serial_2.ips'), + ], log_file_name=os.path.join(tmpdir, 'test.log'), platform_file_name=os.path.join(tmpdir, 'platform.conf'), - debug=None, - verbose_debug=None, + debug=False, + verbose_debug=False, cmd_nodes=0, cmd_ppn=0, ) @@ -124,18 +164,18 @@ def test_basic_serial_multi(tmpdir, capfd): captured = capfd.readouterr() captured_out = captured.out.split('\n') - assert captured_out[0] == "Created " - assert captured_out[1] == "Created " - assert captured_out[2] == "Created " - assert captured_out[3] == "Created " - assert captured_out[4] == "Created " - assert captured_out[5] == "Created " - assert captured_out[7] == "small_worker : init() called" - assert captured_out[9] == "small_worker : init() called" - assert captured_out[11] == "medium_worker : init() called" - assert captured_out[13] == "medium_worker : init() called" - assert captured_out[15] == "large_worker : init() called" - assert captured_out[17] == "large_worker : init() called" + assert captured_out[0] == "Created " + assert captured_out[1] == "Created " + assert captured_out[2] == "Created " + assert captured_out[3] == "Created " + assert captured_out[4] == "Created " + assert captured_out[5] == "Created " + assert captured_out[7] == "SmallWorker : init() called" + assert captured_out[9] == "SmallWorker : init() called" + assert captured_out[11] == "MediumWorker : init() called" + assert captured_out[13] == "MediumWorker : init() called" + assert captured_out[15] == "LargeWorker : init() called" + assert captured_out[17] == "LargeWorker : init() called" assert captured_out[19] == "Current time = 1.00" assert captured_out[20] == "Current time = 1.00" assert captured_out[21] == "Current time = 2.00" @@ -146,13 +186,33 @@ def test_basic_serial_multi(tmpdir, capfd): # check files copied and created for no in ['1', '2']: - driver_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f'test_basic_serial{no}_0/work/drivers_testing_basic_serial*_*/*')))] + driver_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join(f'test_basic_serial_{no}_0/work/drivers_testing_basic_serial*_*/*')) + ) + ] for infile in ['file1', 'ofile1', 'ofile2', 'sfile1', 'sfile2']: assert infile in driver_files - small_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f'test_basic_serial{no}_0/work/workers_testing_small_worker_*/*')))] - medium_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f'test_basic_serial{no}_0/work/workers_testing_medium_worker_*/*')))] - large_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f'test_basic_serial{no}_0/work/workers_testing_large_worker_*/*')))] + small_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join(f'test_basic_serial_{no}_0/work/workers_testing_SmallWorker_*/*')) + ) + ] + medium_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join(f'test_basic_serial_{no}_0/work/workers_testing_MediumWorker_*/*')) + ) + ] + large_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join(f'test_basic_serial_{no}_0/work/workers_testing_LargeWorker_*/*')) + ) + ] if no == '1': for outfile in ['my_out3.50', 'my_out3.60', 'my_out3.70']: @@ -167,81 +227,119 @@ def test_basic_serial_multi(tmpdir, capfd): # check contents of my_out files for outfile in ['my_out3.50', 'my_out3.60', 'my_out3.70']: - for worker in ['workers_testing_small_worker_2', 'workers_testing_medium_worker_3']: - with open(str(tmpdir.join('test_basic_serial1_0/work').join(worker).join(outfile)), 'r') as f: + for worker in [ + 'workers_testing_SmallWorker_2', + 'workers_testing_MediumWorker_3', + ]: + with open( + str(tmpdir.join('test_basic_serial_1_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() assert "results = ['Rank 0 slept for 1.0 seconds']\n" in lines - worker = 'workers_testing_large_worker_4' - with open(str(tmpdir.join('test_basic_serial1_0/work').join(worker).join(outfile)), 'r') as f: + worker = 'workers_testing_LargeWorker_4' + with open( + str(tmpdir.join('test_basic_serial_1_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() - assert "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines + assert ( + "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines + ) for outfile in ['my_out3.40', 'my_out3.50', 'my_out3.60']: - for worker in ['workers_testing_small_worker_6', 'workers_testing_medium_worker_7']: - with open(str(tmpdir.join('test_basic_serial2_0/work').join(worker).join(outfile)), 'r') as f: + for worker in [ + 'workers_testing_SmallWorker_6', + 'workers_testing_MediumWorker_7', + ]: + with open( + str(tmpdir.join('test_basic_serial_2_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() assert "results = ['Rank 0 slept for 1.0 seconds']\n" in lines - worker = 'workers_testing_large_worker_8' - with open(str(tmpdir.join('test_basic_serial2_0/work').join(worker).join(outfile)), 'r') as f: + worker = 'workers_testing_LargeWorker_8' + with open( + str(tmpdir.join('test_basic_serial_2_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() - assert "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines - - # check basic_serial1 sim log file - with open(str(tmpdir.join('test_basic_serial1_0').join('test_basic_serial1_0.log')), 'r') as f: + assert ( + "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines + ) + + # check basic_serial_1 sim log file + with open( + str(tmpdir.join('test_basic_serial_1_0').join('test_basic_serial_1_0.log')), + 'r', + ) as f: lines = f.readlines() # remove timestamp lines = [line[24:] for line in lines] - for worker in ['small_worker_2', 'medium_worker_3', 'large_worker_4']: + for worker in ['SmallWorker_2', 'MediumWorker_3', 'LargeWorker_4']: for timestamp in ['3.50', '3.60', '3.70']: - assert f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' in lines - - # check basic_serial2 sim log file - with open(str(tmpdir.join('test_basic_serial2_0').join('test_basic_serial2_0.log')), 'r') as f: + assert ( + f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' + in lines + ) + + # check basic_serial_2 sim log file + with open( + str(tmpdir.join('test_basic_serial_2_0').join('test_basic_serial_2_0.log')), + 'r', + ) as f: lines = f.readlines() # remove timestamp lines = [line[24:] for line in lines] - for worker in ['small_worker_6', 'medium_worker_7', 'large_worker_8']: + for worker in ['SmallWorker_6', 'MediumWorker_7', 'LargeWorker_8']: for timestamp in ['3.40', '3.50', '3.60']: - assert f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' in lines + assert ( + f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' + in lines + ) # check that the parent_portal_runid is correctly set - serial1_json_files = glob.glob(str(tmpdir.join('test_basic_serial1_0').join('simulation_log').join('*.json'))) + serial1_json_files = glob.glob( + str(tmpdir.join('test_basic_serial_1_0').join('simulation_log').join('*.json')) + ) assert len(serial1_json_files) == 1 with open(serial1_json_files[0], 'r') as json_file: serial1_lines = json_file.readlines() - serial1_IPS_START = json.loads(serial1_lines[0]) - assert serial1_IPS_START['parent_portal_runid'] is None - serial1_portal_runid = serial1_IPS_START['portal_runid'] + serial1_ips_start = json.loads(serial1_lines[0]) + assert serial1_ips_start['parent_portal_runid'] is None + serial1_portal_runid = serial1_ips_start['portal_runid'] - serial2_json_files = glob.glob(str(tmpdir.join('test_basic_serial2_0').join('simulation_log').join('*.json'))) + serial2_json_files = glob.glob( + str(tmpdir.join('test_basic_serial_2_0').join('simulation_log').join('*.json')) + ) assert len(serial2_json_files) == 1 with open(serial2_json_files[0], 'r') as json_file: serial2_lines = json_file.readlines() - serial2_IPS_START = json.loads(serial2_lines[0]) - assert serial2_IPS_START['parent_portal_runid'] == serial1_portal_runid - assert serial2_IPS_START['portal_runid'] is not None - assert serial2_IPS_START['portal_runid'] != serial1_portal_runid + serial2_ips_start = json.loads(serial2_lines[0]) + assert serial2_ips_start['parent_portal_runid'] == serial1_portal_runid + assert serial2_ips_start['portal_runid'] is not None + assert serial2_ips_start['portal_runid'] != serial1_portal_runid @pytest.mark.skipif(not shutil.which('mpirun'), reason='requires mpirun') -def test_basic_concurrent1(tmpdir, capfd): +def test_basic_concurrent_1(tmpdir, capfd): datadir = os.path.dirname(__file__) - copy_config_and_replace('basic_concurrent1.ips', datadir, tmpdir) + copy_config_and_replace('basic_concurrent_1.ips', datadir, tmpdir) shutil.copy(os.path.join(datadir, 'platform.conf'), tmpdir) # setup 'input' files os.system(f'cd {tmpdir}; touch file1 ofile1 ofile2 sfile1 sfile2') framework = Framework( - config_file_list=[os.path.join(tmpdir, 'basic_concurrent1.ips')], + config_file_list=[os.path.join(tmpdir, 'basic_concurrent_1.ips')], log_file_name=os.path.join(tmpdir, 'test.log'), platform_file_name=os.path.join(tmpdir, 'platform.conf'), debug=None, @@ -258,12 +356,12 @@ def test_basic_concurrent1(tmpdir, capfd): captured_err = captured.err.split('\n') assert captured_err[0].startswith('Starting IPS') - assert captured_out[0] == "Created " - assert captured_out[1] == "Created " - assert captured_out[2] == "Created " - assert captured_out[3] == 'small_worker : init() called' - assert captured_out[5] == 'medium_worker : init() called' - assert captured_out[7] == 'large_worker : init() called' + assert captured_out[0] == "Created " + assert captured_out[1] == "Created " + assert captured_out[2] == "Created " + assert captured_out[3] == 'SmallWorker : init() called' + assert captured_out[5] == 'MediumWorker : init() called' + assert captured_out[7] == 'LargeWorker : init() called' assert captured_out[9] == 'Current time = 3.50' assert captured_out[10] == 'nonblocking wait_call() invoked before call 10 finished' assert captured_out[11] == 'Current time = 3.60' @@ -272,13 +370,35 @@ def test_basic_concurrent1(tmpdir, capfd): assert captured_out[14] == 'nonblocking wait_call() invoked before call 16 finished' # check files copied and created - driver_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_concurrent1_0/work/drivers_testing_basic_concurrent1_*/*')))] + driver_files = [ + os.path.basename(f) + for f in glob.glob( + str( + tmpdir.join('test_basic_concurrent_1_0/work/drivers_testing_basic_concurrent_1_*/*') + ) + ) + ] for infile in ['file1', 'ofile1', 'ofile2', 'sfile1', 'sfile2']: assert infile in driver_files - small_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_concurrent1_0/work/workers_testing_small_worker_*/*')))] - medium_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_concurrent1_0/work/workers_testing_medium_worker_*/*')))] - large_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join('test_basic_concurrent1_0/work/workers_testing_large_worker_*/*')))] + small_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join('test_basic_concurrent_1_0/work/workers_testing_SmallWorker_*/*')) + ) + ] + medium_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join('test_basic_concurrent_1_0/work/workers_testing_MediumWorker_*/*')) + ) + ] + large_worker_files = [ + os.path.basename(f) + for f in glob.glob( + str(tmpdir.join('test_basic_concurrent_1_0/work/workers_testing_LargeWorker_*/*')) + ) + ] for outfile in ['my_out3.50', 'my_out3.60', 'my_out3.70']: assert outfile in small_worker_files @@ -287,23 +407,40 @@ def test_basic_concurrent1(tmpdir, capfd): # check contents of my_out files for outfile in ['my_out3.50', 'my_out3.60', 'my_out3.70']: - for worker in ['workers_testing_small_worker_2', 'workers_testing_medium_worker_3']: - with open(str(tmpdir.join('test_basic_concurrent1_0/work').join(worker).join(outfile)), 'r') as f: + for worker in [ + 'workers_testing_SmallWorker_2', + 'workers_testing_MediumWorker_3', + ]: + with open( + str(tmpdir.join('test_basic_concurrent_1_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() assert "results = ['Rank 0 slept for 1.0 seconds']\n" in lines - worker = 'workers_testing_large_worker_4' - with open(str(tmpdir.join('test_basic_concurrent1_0/work').join(worker).join(outfile)), 'r') as f: + worker = 'workers_testing_LargeWorker_4' + with open( + str(tmpdir.join('test_basic_concurrent_1_0/work').join(worker).join(outfile)), + 'r', + ) as f: lines = f.readlines() - assert "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines + assert ( + "results = ['Rank 0 slept for 1.0 seconds', 'Rank 1 slept for 1.0 seconds']\n" in lines + ) # check sim log file - with open(str(tmpdir.join('test_basic_concurrent1_0').join('test_basic_concurrent1_0.log')), 'r') as f: + with open( + str(tmpdir.join('test_basic_concurrent_1_0').join('test_basic_concurrent_1_0.log')), + 'r', + ) as f: lines = f.readlines() # remove timestamp lines = [line[24:] for line in lines] - for worker in ['small_worker_2', 'medium_worker_3', 'large_worker_4']: + for worker in ['SmallWorker_2', 'MediumWorker_3', 'LargeWorker_4']: for timestamp in ['3.50', '3.60', '3.70']: - assert f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' in lines + assert ( + f'workers_testing_{worker} INFO Stepping Worker timestamp={timestamp}\n' + in lines + ) diff --git a/tests/new/test_bad_components.py b/tests/new/test_bad_components.py index 96a74cba..e6dec797 100644 --- a/tests/new/test_bad_components.py +++ b/tests/new/test_bad_components.py @@ -36,7 +36,7 @@ def write_basic_config_and_platform_files(tmpdir, worker): [DRIVER] CLASS = DRIVER SUB_CLASS = - NAME = driver + NAME = Driver BIN_PATH = NPROC = 1 INPUT_FILES = @@ -62,7 +62,9 @@ def write_basic_config_and_platform_files(tmpdir, worker): def test_exception(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, worker='exception_worker') + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, worker='ExceptionWorker' + ) framework = Framework( config_file_list=[str(config_file)], @@ -86,8 +88,8 @@ def test_exception(tmpdir): # remove timestamp lines = [line[24:] for line in lines] - assert 'WORKER__exception_worker_2 ERROR Uncaught Exception in component method.\n' in lines - assert 'DRIVER__driver_1 ERROR Uncaught Exception in component method.\n' in lines + assert 'WORKER__ExceptionWorker_2 ERROR Uncaught Exception in component method.\n' in lines + assert 'DRIVER__Driver_1 ERROR Uncaught Exception in component method.\n' in lines # check event log events = read_event_log(tmpdir) @@ -95,10 +97,13 @@ def test_exception(tmpdir): worker_call_end_event = events[8] - assert worker_call_end_event['code'] == 'DRIVER__driver' + assert worker_call_end_event['code'] == 'DRIVER__Driver' assert worker_call_end_event['eventtype'] == 'IPS_CALL_END' assert not worker_call_end_event['ok'] - assert worker_call_end_event['comment'] == 'Error: "Runtime error" Target = test@exception_worker@2:step(0)' + assert ( + worker_call_end_event['comment'] + == 'Error: "Runtime error" Target = test@exception_worker@2:step(0)' + ) sim_end_event = events[10] assert sim_end_event['code'] == 'Framework' @@ -108,7 +113,9 @@ def test_exception(tmpdir): def test_bad_task(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, worker='bad_task_worker') + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, worker='BadTaskWorker' + ) framework = Framework( config_file_list=[str(config_file)], @@ -132,8 +139,8 @@ def test_bad_task(tmpdir): # remove timestamp lines = [line[24:] for line in lines] - assert 'WORKER__bad_task_worker_2 ERROR Uncaught Exception in component method.\n' in lines - assert 'DRIVER__driver_1 ERROR Uncaught Exception in component method.\n' in lines + assert 'WORKER__BadTaskWorker_2 ERROR Uncaught Exception in component method.\n' in lines + assert 'DRIVER__Driver_1 ERROR Uncaught Exception in component method.\n' in lines # check event log events = read_event_log(tmpdir) @@ -141,10 +148,13 @@ def test_bad_task(tmpdir): worker_call_end_event = events[8] - assert worker_call_end_event['code'] == 'DRIVER__driver' + assert worker_call_end_event['code'] == 'DRIVER__Driver' assert worker_call_end_event['eventtype'] == 'IPS_CALL_END' assert not worker_call_end_event['ok'] - assert worker_call_end_event['comment'] == 'Error: "task binary of wrong type, expected str but found int" Target = test@bad_task_worker@2:step(0)' + assert ( + worker_call_end_event['comment'] + == 'Error: "task binary of wrong type, expected str but found int" Target = test@bad_task_worker@2:step(0)' + ) sim_end_event = events[10] assert sim_end_event['code'] == 'Framework' @@ -154,7 +164,9 @@ def test_bad_task(tmpdir): def test_bad_task_pool1(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, worker='bad_task_pool_worker1') + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, worker='BadTaskPoolWorker1' + ) framework = Framework( config_file_list=[str(config_file)], @@ -178,12 +190,16 @@ def test_bad_task_pool1(tmpdir): # remove timestamp lines = [line[24:] for line in lines] - assert 'WORKER__bad_task_pool_worker1_2 ERROR Uncaught Exception in component method.\n' in lines - assert 'DRIVER__driver_1 ERROR Uncaught Exception in component method.\n' in lines + assert ( + 'WORKER__BadTaskPoolWorker1_2 ERROR Uncaught Exception in component method.\n' in lines + ) + assert 'DRIVER__Driver_1 ERROR Uncaught Exception in component method.\n' in lines def test_bad_task_pool2(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, worker='bad_task_pool_worker2') + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, worker='BadTaskPoolWorker2' + ) framework = Framework( config_file_list=[str(config_file)], @@ -207,12 +223,16 @@ def test_bad_task_pool2(tmpdir): # remove timestamp lines = [line[24:] for line in lines] - assert 'WORKER__bad_task_pool_worker2_2 ERROR Uncaught Exception in component method.\n' in lines - assert 'DRIVER__driver_1 ERROR Uncaught Exception in component method.\n' in lines + assert ( + 'WORKER__BadTaskPoolWorker2_2 ERROR Uncaught Exception in component method.\n' in lines + ) + assert 'DRIVER__Driver_1 ERROR Uncaught Exception in component method.\n' in lines def test_assign_protected_attribute(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, worker='assign_protected_attribute') + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, worker='AssignProtectedAttribute' + ) framework = Framework( config_file_list=[str(config_file)], @@ -234,19 +254,24 @@ def test_assign_protected_attribute(tmpdir): assert ( "AttributeError: can't set attribute\n" in lines or "AttributeError: can't set attribute 'args'\n" in lines - or "AttributeError: property 'args' of 'assign_protected_attribute' object has no setter\n" in lines + or "AttributeError: property 'args' of 'assign_protected_attribute' object has no setter\n" + in lines ) assert ( "Exception: can't set attribute\n" in lines or "Exception: can't set attribute 'args'\n" in lines - or "Exception: property 'args' of 'assign_protected_attribute' object has no setter\n" in lines + or "Exception: property 'args' of 'assign_protected_attribute' object has no setter\n" + in lines ) # remove timestamp lines = [line[24:] for line in lines] - assert 'WORKER__assign_protected_attribute_2 ERROR Uncaught Exception in component method.\n' in lines - assert 'DRIVER__driver_1 ERROR Uncaught Exception in component method.\n' in lines + assert ( + 'WORKER__AssignProtectedAttribute_2 ERROR Uncaught Exception in component method.\n' + in lines + ) + assert 'DRIVER__Driver_1 ERROR Uncaught Exception in component method.\n' in lines # check event log events = read_event_log(tmpdir) @@ -254,7 +279,7 @@ def test_assign_protected_attribute(tmpdir): worker_call_end_event = events[8] - assert worker_call_end_event['code'] == 'DRIVER__driver' + assert worker_call_end_event['code'] == 'DRIVER__Driver' assert worker_call_end_event['eventtype'] == 'IPS_CALL_END' assert not worker_call_end_event['ok'] # python 3.10 and 3.11 have different error messages @@ -272,7 +297,9 @@ def test_assign_protected_attribute(tmpdir): def read_event_log(tmpdir): - sim_event_log_json = next(f for f in os.listdir(tmpdir.join('simulation_log')) if f.endswith('.json')) + sim_event_log_json = next( + f for f in os.listdir(tmpdir.join('simulation_log')) if f.endswith('.json') + ) with open(str(tmpdir.join('simulation_log').join(sim_event_log_json)), 'r') as f: lines = f.readlines() diff --git a/tests/new/test_component_logging.py b/tests/new/test_component_logging.py index 961c97f7..3ac092be 100644 --- a/tests/new/test_component_logging.py +++ b/tests/new/test_component_logging.py @@ -1,6 +1,14 @@ from ipsframework import Framework -map_log_to_level = {'log': 'INFO', 'debug': 'DEBUG', 'info': 'INFO', 'warning': 'WARNING', 'error': 'ERROR', 'exception': 'ERROR', 'critical': 'CRITICAL'} +map_log_to_level = { + 'log': 'INFO', + 'debug': 'DEBUG', + 'info': 'INFO', + 'warning': 'WARNING', + 'error': 'ERROR', + 'exception': 'ERROR', + 'critical': 'CRITICAL', +} def write_basic_config_and_platform_files(tmpdir, debug=False): @@ -37,7 +45,7 @@ def write_basic_config_and_platform_files(tmpdir, debug=False): [LOGGING_DRIVER] CLASS = LOGGING SUB_CLASS = - NAME = logging_tester + NAME = LoggingTester BIN_PATH = NPROC = 1 INPUT_FILES = @@ -74,29 +82,40 @@ def test_component_logging(tmpdir): # remove timestamp lines = [line[24:] for line in lines] - component_id = 'LOGGING__logging_tester_1' + component_id = 'LOGGING__loggingTester_1' # for log_level=WARNING only WARNING, ERROR and CRITICAL logs should be included # DEBUG and INFO should be excluded for method in ['init', 'step', 'finalize']: for log_type in ['warning', 'error', 'exception', 'critical']: - assert f'{component_id} {map_log_to_level[log_type]:8} {method} msg: {log_type}\n' in lines + assert ( + f'{component_id} {map_log_to_level[log_type]:8} {method} msg: {log_type}\n' in lines + ) for log_type in ['log', 'debug', 'info']: - assert f'{component_id} {map_log_to_level[log_type]:8} {method} msg: {log_type}\n' not in lines + assert ( + f'{component_id} {map_log_to_level[log_type]:8} {method} msg: {log_type}\n' + not in lines + ) # check message formatting with arguments for log_type in ['warning', 'error', 'exception', 'critical']: - assert f'{component_id} {map_log_to_level[log_type]:8} step msg: {log_type} timestamp=0 test\n' in lines + assert ( + f'{component_id} {map_log_to_level[log_type]:8} step msg: {log_type} timestamp=0 test\n' + in lines + ) for log_type in ['log', 'debug', 'info']: - assert f'{component_id} {map_log_to_level[log_type]:8} step msg: {log_type} timestamp=0 test\n' not in lines + assert ( + f'{component_id} {map_log_to_level[log_type]:8} step msg: {log_type} timestamp=0 test\n' + not in lines + ) # check stdout redirect with open(str(tmpdir.join('test.out')), 'r') as f: lines = f.readlines() - assert lines[0] == 'test@logging_tester@1.init\n' - assert lines[1] == 'test@logging_tester@1.step\n' - assert lines[2] == 'test@logging_tester@1.finalize\n' + assert lines[0] == 'test@LoggingTester@1.init\n' + assert lines[1] == 'test@LoggingTester@1.step\n' + assert lines[2] == 'test@LoggingTester@1.finalize\n' def test_component_logging_debug(tmpdir): @@ -121,13 +140,34 @@ def test_component_logging_debug(tmpdir): # remove timestamp lines = [line[24:] for line in lines] - component_id = 'LOGGING__logging_tester_1' + component_id = 'LOGGING__LoggingTester_1' # for log_level=DEBUG all logs should be included for method in ['init', 'step', 'finalize']: - for log_type in ['log', 'debug', 'info', 'warning', 'error', 'exception', 'critical']: - assert f'{component_id} {map_log_to_level[log_type]:8} {method} msg: {log_type}\n' in lines + for log_type in [ + 'log', + 'debug', + 'info', + 'warning', + 'error', + 'exception', + 'critical', + ]: + assert ( + f'{component_id} {map_log_to_level[log_type]:8} {method} msg: {log_type}\n' in lines + ) # check message formatting with arguments - for log_type in ['log', 'debug', 'info', 'warning', 'error', 'exception', 'critical']: - assert f'{component_id} {map_log_to_level[log_type]:8} step msg: {log_type} timestamp=0 test\n' in lines + for log_type in [ + 'log', + 'debug', + 'info', + 'warning', + 'error', + 'exception', + 'critical', + ]: + assert ( + f'{component_id} {map_log_to_level[log_type]:8} step msg: {log_type} timestamp=0 test\n' + in lines + ) diff --git a/tests/new/test_cori_srun.py b/tests/new/test_cori_srun.py index 85b3963f..87ba2e65 100644 --- a/tests/new/test_cori_srun.py +++ b/tests/new/test_cori_srun.py @@ -52,7 +52,7 @@ def write_basic_config_and_platform_files(tmpdir, name): @pytest.mark.cori def test_srun_openmp_on_cori(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, 'openmp_task') + platform_file, config_file = write_basic_config_and_platform_files(tmpdir, 'OpenmpTask') framework = Framework( config_file_list=[str(config_file)], @@ -70,78 +70,159 @@ def test_srun_openmp_on_cori(tmpdir): json_files = glob.glob(str(tmpdir.join('simulation_log').join('*.json'))) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: - comments = [json.loads(line)['comment'].split(', ', maxsplit=4)[3:] for line in json_file.readlines()] + comments = [json.loads(line)['comment'].split(', ', maxsplit=4)[3:] for line in json_file] # check that the process output log files are created work_dir = tmpdir.join('work').join('OPENMP__openmp_task_1') # 0 for c in (5, 7, 9): - assert comments[c][0] == 'Target = srun -N 1 -n 1 -c 32 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' - assert comments[c][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '32'}" + assert ( + comments[c][0] + == 'Target = srun -N 1 -n 1 -c 32 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) + assert ( + comments[c][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '32'}" + ) for log in ('01', '02', '03'): lines = sorted(work_dir.join(f'log.{log}').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0-63)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0-63)\n' + ) # 1 for c in (11, 13, 15): - assert comments[c][0] == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' - assert comments[c][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + assert ( + comments[c][0] + == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) + assert ( + comments[c][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + ) for log in ('11', '12', '13'): lines = sorted(work_dir.join(f'log.{log}').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0-7,32-39)\n') - assert lines[1].startswith('Hello from rank 1') and lines[1].endswith('(core affinity = 16-23,48-55)\n') - assert lines[2].startswith('Hello from rank 2') and lines[2].endswith('(core affinity = 8-15,40-47)\n') - assert lines[3].startswith('Hello from rank 3') and lines[3].endswith('(core affinity = 24-31,56-63)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0-7,32-39)\n' + ) + assert lines[1].startswith('Hello from rank 1') and lines[1].endswith( + '(core affinity = 16-23,48-55)\n' + ) + assert lines[2].startswith('Hello from rank 2') and lines[2].endswith( + '(core affinity = 8-15,40-47)\n' + ) + assert lines[3].startswith('Hello from rank 3') and lines[3].endswith( + '(core affinity = 24-31,56-63)\n' + ) # 2 for c in (17, 19, 21): - assert comments[c][0] == 'Target = srun -N 1 -n 32 -c 1 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' - assert comments[c][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '1'}" + assert ( + comments[c][0] + == 'Target = srun -N 1 -n 32 -c 1 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) + assert ( + comments[c][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '1'}" + ) for log in ('21', '22', '23'): - lines = sorted(work_dir.join(f'log.{log}').readlines(), key=lambda a: int(a.split()[3].replace(',', ''))) + lines = sorted( + work_dir.join(f'log.{log}').readlines(), + key=lambda a: int(a.split()[3].replace(',', '')), + ) for n, line in enumerate(lines): cores = n // 2 + n % 2 * 16 - assert line.startswith(f'Hello from rank {n}') and line.endswith(f'(core affinity = {cores},{cores + 32})\n') + assert line.startswith(f'Hello from rank {n}') and line.endswith( + f'(core affinity = {cores},{cores + 32})\n' + ) # 31 - assert comments[23][0] == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' - assert comments[23][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + assert ( + comments[23][0] + == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) + assert ( + comments[23][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + ) lines = sorted(work_dir.join('log.31').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0-7,32-39)\n') - assert lines[1].startswith('Hello from rank 1') and lines[1].endswith('(core affinity = 16-23,48-55)\n') - assert lines[2].startswith('Hello from rank 2') and lines[2].endswith('(core affinity = 8-15,40-47)\n') - assert lines[3].startswith('Hello from rank 3') and lines[3].endswith('(core affinity = 24-31,56-63)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0-7,32-39)\n' + ) + assert lines[1].startswith('Hello from rank 1') and lines[1].endswith( + '(core affinity = 16-23,48-55)\n' + ) + assert lines[2].startswith('Hello from rank 2') and lines[2].endswith( + '(core affinity = 8-15,40-47)\n' + ) + assert lines[3].startswith('Hello from rank 3') and lines[3].endswith( + '(core affinity = 24-31,56-63)\n' + ) # 32 - assert comments[25][0] == 'Target = srun -N 1 -n 4 -c 4 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' - assert comments[25][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '4'}" + assert ( + comments[25][0] + == 'Target = srun -N 1 -n 4 -c 4 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) + assert ( + comments[25][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '4'}" + ) lines = sorted(work_dir.join('log.32').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0-3,32-35)\n') - assert lines[1].startswith('Hello from rank 1') and lines[1].endswith('(core affinity = 16-19,48-51)\n') - assert lines[2].startswith('Hello from rank 2') and lines[2].endswith('(core affinity = 4-7,36-39)\n') - assert lines[3].startswith('Hello from rank 3') and lines[3].endswith('(core affinity = 20-23,52-55)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0-3,32-35)\n' + ) + assert lines[1].startswith('Hello from rank 1') and lines[1].endswith( + '(core affinity = 16-19,48-51)\n' + ) + assert lines[2].startswith('Hello from rank 2') and lines[2].endswith( + '(core affinity = 4-7,36-39)\n' + ) + assert lines[3].startswith('Hello from rank 3') and lines[3].endswith( + '(core affinity = 20-23,52-55)\n' + ) # 33 - assert comments[27][0] == 'Target = srun -N 1 -n 4 -c 2 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' - assert comments[27][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '2'}" + assert ( + comments[27][0] + == 'Target = srun -N 1 -n 4 -c 2 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) + assert ( + comments[27][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '2'}" + ) lines = sorted(work_dir.join('log.33').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0,1,32,33)\n') - assert lines[1].startswith('Hello from rank 1') and lines[1].endswith('(core affinity = 16,17,48,49)\n') - assert lines[2].startswith('Hello from rank 2') and lines[2].endswith('(core affinity = 2,3,34,35)\n') - assert lines[3].startswith('Hello from rank 3') and lines[3].endswith('(core affinity = 18,19,50,51)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0,1,32,33)\n' + ) + assert lines[1].startswith('Hello from rank 1') and lines[1].endswith( + '(core affinity = 16,17,48,49)\n' + ) + assert lines[2].startswith('Hello from rank 2') and lines[2].endswith( + '(core affinity = 2,3,34,35)\n' + ) + assert lines[3].startswith('Hello from rank 3') and lines[3].endswith( + '(core affinity = 18,19,50,51)\n' + ) # openmp # 41 - assert comments[29][0] == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-hybrid.gnu.cori ' - assert comments[29][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + assert ( + comments[29][0] + == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-hybrid.gnu.cori ' + ) + assert ( + comments[29][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + ) lines = sorted(work_dir.join('log.41').readlines()) for n, line in enumerate(lines): @@ -149,8 +230,14 @@ def test_srun_openmp_on_cori(tmpdir): assert line.endswith(f'(core affinity = {n % 8 + n // 16 * 8 + n // 8 % 2 * 16})\n') # 42 - assert comments[31][0] == 'Target = srun -N 1 -n 4 -c 4 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-hybrid.gnu.cori ' - assert comments[31][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '4'}" + assert ( + comments[31][0] + == 'Target = srun -N 1 -n 4 -c 4 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-hybrid.gnu.cori ' + ) + assert ( + comments[31][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '4'}" + ) lines = sorted(work_dir.join('log.42').readlines()) for n, line in enumerate(lines): @@ -158,8 +245,14 @@ def test_srun_openmp_on_cori(tmpdir): assert line.endswith(f'(core affinity = {n % 4 + n // 8 * 4 + n // 4 % 2 * 16})\n') # 43 - assert comments[33][0] == 'Target = srun -N 1 -n 4 -c 2 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-hybrid.gnu.cori ' - assert comments[33][1] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '2'}" + assert ( + comments[33][0] + == 'Target = srun -N 1 -n 4 -c 2 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-hybrid.gnu.cori ' + ) + assert ( + comments[33][1] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '2'}" + ) lines = sorted(work_dir.join('log.43').readlines()) for n, line in enumerate(lines): @@ -169,7 +262,7 @@ def test_srun_openmp_on_cori(tmpdir): @pytest.mark.cori def test_srun_openmp_on_cori_pool(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, 'openmp_task_pool') + platform_file, config_file = write_basic_config_and_platform_files(tmpdir, 'OpenmpTaskPool') framework = Framework( config_file_list=[str(config_file)], @@ -187,40 +280,82 @@ def test_srun_openmp_on_cori_pool(tmpdir): json_files = glob.glob(str(tmpdir.join('simulation_log').join('*.json'))) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: - comments = [json.loads(line)['comment'].split(', ', maxsplit=5)[3:] for line in json_file.readlines()] + comments = [json.loads(line)['comment'].split(', ', maxsplit=5)[3:] for line in json_file] # check that the process output log files are created work_dir = tmpdir.join('work').join('OPENMP__openmp_task_pool_1') # 1 - assert comments[6][0] == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + assert ( + comments[6][0] + == 'Target = srun -N 1 -n 4 -c 8 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) assert comments[6][1] == 'task_name = task_1' - assert comments[6][2] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + assert ( + comments[6][2] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '8'}" + ) lines = sorted(work_dir.join('log.1').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0-7,32-39)\n') - assert lines[1].startswith('Hello from rank 1') and lines[1].endswith('(core affinity = 16-23,48-55)\n') - assert lines[2].startswith('Hello from rank 2') and lines[2].endswith('(core affinity = 8-15,40-47)\n') - assert lines[3].startswith('Hello from rank 3') and lines[3].endswith('(core affinity = 24-31,56-63)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0-7,32-39)\n' + ) + assert lines[1].startswith('Hello from rank 1') and lines[1].endswith( + '(core affinity = 16-23,48-55)\n' + ) + assert lines[2].startswith('Hello from rank 2') and lines[2].endswith( + '(core affinity = 8-15,40-47)\n' + ) + assert lines[3].startswith('Hello from rank 3') and lines[3].endswith( + '(core affinity = 24-31,56-63)\n' + ) # 2 - assert comments[8][0] == 'Target = srun -N 1 -n 4 -c 4 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + assert ( + comments[8][0] + == 'Target = srun -N 1 -n 4 -c 4 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) assert comments[8][1] == 'task_name = task_2' - assert comments[8][2] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '4'}" + assert ( + comments[8][2] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '4'}" + ) lines = sorted(work_dir.join('log.2').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0-3,32-35)\n') - assert lines[1].startswith('Hello from rank 1') and lines[1].endswith('(core affinity = 16-19,48-51)\n') - assert lines[2].startswith('Hello from rank 2') and lines[2].endswith('(core affinity = 4-7,36-39)\n') - assert lines[3].startswith('Hello from rank 3') and lines[3].endswith('(core affinity = 20-23,52-55)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0-3,32-35)\n' + ) + assert lines[1].startswith('Hello from rank 1') and lines[1].endswith( + '(core affinity = 16-19,48-51)\n' + ) + assert lines[2].startswith('Hello from rank 2') and lines[2].endswith( + '(core affinity = 4-7,36-39)\n' + ) + assert lines[3].startswith('Hello from rank 3') and lines[3].endswith( + '(core affinity = 20-23,52-55)\n' + ) # 3 - assert comments[10][0] == 'Target = srun -N 1 -n 4 -c 2 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + assert ( + comments[10][0] + == 'Target = srun -N 1 -n 4 -c 2 --threads-per-core=1 --cpu-bind=cores /usr/common/software/bin/check-mpi.gnu.cori ' + ) assert comments[10][1] == 'task_name = task_3' - assert comments[10][2] == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '2'}" + assert ( + comments[10][2] + == "env = {'OMP_PLACES': 'threads', 'OMP_PROC_BIND': 'spread', 'OMP_NUM_THREADS': '2'}" + ) lines = sorted(work_dir.join('log.3').readlines()) - assert lines[0].startswith('Hello from rank 0') and lines[0].endswith('(core affinity = 0,1,32,33)\n') - assert lines[1].startswith('Hello from rank 1') and lines[1].endswith('(core affinity = 16,17,48,49)\n') - assert lines[2].startswith('Hello from rank 2') and lines[2].endswith('(core affinity = 2,3,34,35)\n') - assert lines[3].startswith('Hello from rank 3') and lines[3].endswith('(core affinity = 18,19,50,51)\n') + assert lines[0].startswith('Hello from rank 0') and lines[0].endswith( + '(core affinity = 0,1,32,33)\n' + ) + assert lines[1].startswith('Hello from rank 1') and lines[1].endswith( + '(core affinity = 16,17,48,49)\n' + ) + assert lines[2].startswith('Hello from rank 2') and lines[2].endswith( + '(core affinity = 2,3,34,35)\n' + ) + assert lines[3].startswith('Hello from rank 3') and lines[3].endswith( + '(core affinity = 18,19,50,51)\n' + ) diff --git a/tests/new/test_dask.py b/tests/new/test_dask.py index a763dae1..38239f66 100644 --- a/tests/new/test_dask.py +++ b/tests/new/test_dask.py @@ -10,7 +10,17 @@ from ipsframework import Framework -def write_basic_config_and_platform_files(tmpdir, timeout='', logfile='', errfile='', nproc=1, exe='/bin/sleep', value='', shifter=False, gpus=0): +def write_basic_config_and_platform_files( + tmpdir, + timeout='', + logfile='', + errfile='', + nproc=1, + exe='/bin/sleep', + value='', + shifter=False, + gpus=0, +): platform_file = tmpdir.join('platform.conf') platform = f"""MPIRUN = eval @@ -44,7 +54,7 @@ def write_basic_config_and_platform_files(tmpdir, timeout='', logfile='', errfil [DRIVER] CLASS = DRIVER SUB_CLASS = - NAME = driver + NAME = Driver BIN_PATH = NPROC = 1 INPUT_FILES = @@ -54,7 +64,7 @@ def write_basic_config_and_platform_files(tmpdir, timeout='', logfile='', errfil [DASK] CLASS = DASK SUB_CLASS = - NAME = dask_worker + NAME = DaskWorker BIN_PATH = EXECUTABLE = {exe} VALUE = {value} @@ -118,18 +128,26 @@ def test_dask(tmpdir): assert eventtypes.count('IPS_LAUNCH_DASK_TASK') == 4 assert eventtypes.count('IPS_TASK_END') == 5 - launch_dask_comments = [e.get('comment') for e in lines if e.get('eventtype') == 'IPS_LAUNCH_DASK_TASK'] + launch_dask_comments = [ + e.get('comment') for e in lines if e.get('eventtype') == 'IPS_LAUNCH_DASK_TASK' + ] for task in range(4): assert f'task_name = task_{task}, Target = /bin/sleep 1' in launch_dask_comments - task_end_comments = [e.get('comment')[:-4] for e in lines if e.get('eventtype') == 'IPS_TASK_END'] + task_end_comments = [ + e.get('comment')[:-4] for e in lines if e.get('eventtype') == 'IPS_TASK_END' + ] for task in range(4): assert f'task_name = task_{task}, elapsed time = 1' in task_end_comments -@pytest.mark.skipif(shutil.which('shifter') is not None, reason="This tests only works if shifter doesn't exist") +@pytest.mark.skipif( + shutil.which('shifter') is not None, reason="This tests only works if shifter doesn't exist" +) def test_dask_shifter_fail(tmpdir): - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, value=1, shifter=True) + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, value=1, shifter=True + ) framework = Framework( config_file_list=[str(config_file)], @@ -150,7 +168,10 @@ def test_dask_shifter_fail(tmpdir): # remove timestamp lines = [line[24:] for line in lines] - assert 'DASK__dask_worker_2 ERROR Requested to run dask within shifter but shifter not available\n' in lines + assert ( + 'DASK__dask_worker_2 ERROR Requested to run dask within shifter but shifter not available\n' + in lines + ) # check simulation_log, make sure it includes events from dask tasks json_files = glob.glob(str(tmpdir.join('simulation_log').join('*.json'))) @@ -169,12 +190,14 @@ def test_dask_fake_shifter(tmpdir, monkeypatch): shifter.write('#!/bin/bash\necho Running $@ in shifter >> shifter.log\n$@\n') shifter.chmod(448) # 700 - old_PATH = os.environ['PATH'] + old_path = os.environ['PATH'] monkeypatch.setenv('PATH', str(tmpdir), prepend=os.pathsep) # need to reimport to get fake shifter importlib.reload(ipsframework.services) - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, value=1, shifter=True) + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, value=1, shifter=True + ) framework = Framework( config_file_list=[str(config_file)], @@ -188,7 +211,7 @@ def test_dask_fake_shifter(tmpdir, monkeypatch): framework.run() - monkeypatch.setenv('PATH', old_PATH) + monkeypatch.setenv('PATH', old_path) # need to reimport to remove fake shifter importlib.reload(ipsframework.services) @@ -219,11 +242,15 @@ def test_dask_fake_shifter(tmpdir, monkeypatch): assert eventtypes.count('IPS_LAUNCH_DASK_TASK') == 4 assert eventtypes.count('IPS_TASK_END') == 5 - launch_dask_comments = [e.get('comment') for e in lines if e.get('eventtype') == 'IPS_LAUNCH_DASK_TASK'] + launch_dask_comments = [ + e.get('comment') for e in lines if e.get('eventtype') == 'IPS_LAUNCH_DASK_TASK' + ] for task in range(4): assert f'task_name = task_{task}, Target = /bin/sleep 1' in launch_dask_comments - task_end_comments = [e.get('comment')[:-4] for e in lines if e.get('eventtype') == 'IPS_TASK_END'] + task_end_comments = [ + e.get('comment')[:-4] for e in lines if e.get('eventtype') == 'IPS_TASK_END' + ] for task in range(4): assert f'task_name = task_{task}, elapsed time = 1' in task_end_comments @@ -279,7 +306,9 @@ def test_dask_timeout(tmpdir): assert eventtypes.count('IPS_LAUNCH_DASK_TASK') == 4 assert eventtypes.count('IPS_TASK_END') == 5 - launch_dask_comments = [e.get('comment') for e in lines if e.get('eventtype') == 'IPS_LAUNCH_DASK_TASK'] + launch_dask_comments = [ + e.get('comment') for e in lines if e.get('eventtype') == 'IPS_LAUNCH_DASK_TASK' + ] for task in range(4): assert f'task_name = task_{task}, Target = /bin/sleep 100' in launch_dask_comments @@ -321,7 +350,10 @@ def test_dask_nproc(tmpdir): assert log.format(f'task_{i} 0') in lines # check for warning message that dask isn't being used - assert 'DASK__dask_worker_2 WARNING Requested use_dask but cannot because multiple processors requested\n' in lines + assert ( + 'DASK__dask_worker_2 WARNING Requested use_dask but cannot because multiple processors requested\n' + in lines + ) def test_dask_logfile(tmpdir): @@ -329,7 +361,9 @@ def test_dask_logfile(tmpdir): exe.write('#!/bin/bash\necho Running $1\n>&2 echo ERROR $1\n') exe.chmod(448) # 700 - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, exe=str(exe), logfile='task_{}.log') + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, exe=str(exe), logfile='task_{}.log' + ) framework = Framework( config_file_list=[str(config_file)], @@ -373,7 +407,9 @@ def test_dask_logfile_errfile(tmpdir): exe = tmpdir.join('stdouterr_write.sh') exe.write('#!/bin/bash\necho Running $1\n>&2 echo ERROR $1\n') exe.chmod(448) # 700 - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, exe=str(exe), logfile='task_{}.log', errfile='task_{}.err') + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, exe=str(exe), logfile='task_{}.log', errfile='task_{}.err' + ) framework = Framework( config_file_list=[str(config_file)], @@ -425,10 +461,14 @@ def test_dask_shifter_on_cori(tmpdir): #SBATCH --image=continuumio/anaconda3:2020.11 """ exe = tmpdir.join('shifter_env.sh') - exe.write('#!/bin/bash\necho Running $1\necho SHIFTER_RUNTIME=$SHIFTER_RUNTIME\necho SHIFTER_IMAGEREQUEST=$SHIFTER_IMAGEREQUEST\n') + exe.write( + '#!/bin/bash\necho Running $1\necho SHIFTER_RUNTIME=$SHIFTER_RUNTIME\necho SHIFTER_IMAGEREQUEST=$SHIFTER_IMAGEREQUEST\n' + ) exe.chmod(448) # 700 - platform_file, config_file = write_basic_config_and_platform_files(tmpdir, exe=str(exe), logfile='task_{}.log', shifter=True) + platform_file, config_file = write_basic_config_and_platform_files( + tmpdir, exe=str(exe), logfile='task_{}.log', shifter=True + ) framework = Framework( config_file_list=[str(config_file)], @@ -502,7 +542,7 @@ def test_dask_with_1_gpu(tmpdir): json_files = glob.glob(str(tmpdir.join('simulation_log').join('*.json'))) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: - comments = [json.loads(line)['comment'].split(', ', maxsplit=5)[2:] for line in json_file.readlines()] + comments = [json.loads(line)['comment'].split(', ', maxsplit=5)[2:] for line in json_file] assert comments[10][0] == 'nproc = 1 ' assert comments[10][1].startswith('Target = ') @@ -543,7 +583,7 @@ def test_dask_with_2_gpus(tmpdir): json_files = glob.glob(str(tmpdir.join('simulation_log').join('*.json'))) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: - comments = [json.loads(line)['comment'].split(', ', maxsplit=5)[2:] for line in json_file.readlines()] + comments = [json.loads(line)['comment'].split(', ', maxsplit=5)[2:] for line in json_file] assert comments[10][0] == 'nproc = 2 ' assert comments[10][1].startswith('Target = ') diff --git a/tests/new/test_dataManager_state.py b/tests/new/test_data_manager_state.py similarity index 68% rename from tests/new/test_dataManager_state.py rename to tests/new/test_data_manager_state.py index 9dee75d6..d63679f7 100644 --- a/tests/new/test_dataManager_state.py +++ b/tests/new/test_data_manager_state.py @@ -38,23 +38,23 @@ def write_basic_config_and_platform_files(tmpdir): [init] CLASS = DATA_INIT SUB_CLASS = - NAME = init_dataManager + NAME = InitDataManager BIN_PATH = NPROC = 1 INPUT_FILES = OUTPUT_FILES = SCRIPT = - MODULE = components.drivers.init_dataManager + MODULE = components.drivers.init_data_manager [driver] CLASS = DATA_DRIVER SUB_CLASS = - NAME = driver_dataManager + NAME = DriverDataManager BIN_PATH = NPROC = 1 INPUT_FILES = OUTPUT_FILES = SCRIPT = - MODULE = components.drivers.driver_dataManager + MODULE = components.drivers.driver_data_manager """ with open(config_file, 'w') as f: @@ -63,7 +63,7 @@ def write_basic_config_and_platform_files(tmpdir): return platform_file, config_file -def test_dataManager_state_file(tmpdir): +def test_data_manager_state_file(tmpdir): platform_file, config_file = write_basic_config_and_platform_files(tmpdir) framework = Framework( @@ -80,16 +80,20 @@ def test_dataManager_state_file(tmpdir): # check output files exist for filename in ['state.dat', 'state100.dat']: - assert os.path.exists(str(tmpdir.join('work').join('DATA_INIT__init_dataManager_1').join(filename))) - assert os.path.exists(str(tmpdir.join('work').join('DATA_DRIVER__driver_dataManager_2').join(filename))) + assert os.path.exists( + str(tmpdir.join('work').join('DATA_INIT__InitDataManager_1').join(filename)) + ) + assert os.path.exists( + str(tmpdir.join('work').join('DATA_DRIVER__DriverDataManager_2').join(filename)) + ) assert os.path.exists(str(tmpdir.join('work').join('state').join(filename))) # check output log file test_map = ( - ('DATA_INIT__init_dataManager_1', 'state.dat', 1), - ('DATA_INIT__init_dataManager_1', 'state100.dat', 100), - ('DATA_DRIVER__driver_dataManager_2', 'state.dat', 2), - ('DATA_DRIVER__driver_dataManager_2', 'state100.dat', 101), + ('DATA_INIT__InitDataManager_1', 'state.dat', 1), + ('DATA_INIT__InitDataManager_1', 'state100.dat', 100), + ('DATA_DRIVER__DriverDataManager_2', 'state.dat', 2), + ('DATA_DRIVER__DriverDataManager_2', 'state100.dat', 101), ('state', 'state.dat', 2), ('state', 'state100.dat', 101), ) @@ -99,8 +103,13 @@ def test_dataManager_state_file(tmpdir): assert value == result # check merge_current_state logfile - logfile = str(tmpdir.join('work').join('DATA_DRIVER__driver_dataManager_2').join('merge_current_state.log')) + logfile = str( + tmpdir.join('work').join('DATA_DRIVER__DriverDataManager_2').join('merge_current_state.log') + ) assert os.path.exists(logfile) # remove tmpdir from log output log = open(logfile).readline().replace(str(tmpdir), '') - assert log == '-input /work/state/state.dat -updates /work/DATA_DRIVER__driver_dataManager_2/partial_state_file\n' + assert ( + log + == '-input /work/state/state.dat -updates /work/DATA_DRIVER__DriverDataManager_2/partial_state_file\n' + ) diff --git a/tests/new/test_ips_framework.py b/tests/new/test_ips_framework.py index 1139d975..c5552ca8 100644 --- a/tests/new/test_ips_framework.py +++ b/tests/new/test_ips_framework.py @@ -9,8 +9,7 @@ def write_basic_config_and_platform_files(tmpdir): test_component = tmpdir.join('test_component.py') - driver = """#!/usr/bin/env python3 -from ipsframework.component import Component + driver = """from ipsframework.component import Component class test_driver(Component): def __init__(self, services, config): super().__init__(services, config) @@ -47,7 +46,7 @@ def __init__(self, services, config): [test_driver] CLASS = driver SUB_CLASS = - NAME = test_driver + NAME = TestDriver NPROC = 1 BIN_PATH = INPUT_DIR = @@ -94,12 +93,12 @@ def test_framework_simple(tmpdir, capfd): # check all registered service handlers service_handlers = sorted(framework.service_handler.keys()) assert service_handlers == [ - 'createListener', + 'create_listener', 'create_simulation', - 'existsTopic', + 'exists_topic', 'finish_task', - 'getSubscription', - 'getTopic', + 'get_subscription', + 'get_topic', 'get_allocation', 'get_config_parameter', 'get_port', @@ -109,16 +108,16 @@ def test_framework_simple(tmpdir, capfd): 'init_task_pool', 'launch_task', 'merge_current_plasma_state', - 'processEvents', - 'registerEventListener', - 'registerSubscriber', + 'process_events', + 'register_event_listener', + 'register_subscriber', 'release_allocation', - 'removeSubscription', - 'sendEvent', + 'remove_subscription', + 'send_event', 'set_config_parameter', 'stage_state', - 'unregisterEventListener', - 'unregisterSubscriber', + 'unregister_event_listener', + 'unregister_subscriber', 'update_state', 'wait_call', ] diff --git a/tests/new/test_ips_main.py b/tests/new/test_ips_main.py index dd09707d..9e145594 100644 --- a/tests/new/test_ips_main.py +++ b/tests/new/test_ips_main.py @@ -8,65 +8,75 @@ @mock.patch('ipsframework.ips.Framework') -def test_ips_main(MockFramework): +def test_ips_main(mock_framework): # override sys.argv for testing sys.argv = ['ips.py'] with pytest.raises(SystemExit) as excinfo: ips.main() assert excinfo.value.code == 2 - MockFramework.assert_not_called() + mock_framework.assert_not_called() - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--simulation=sim.cfg'] with pytest.raises(SystemExit) as excinfo: ips.main() assert excinfo.value.code == 2 - MockFramework.assert_not_called() + mock_framework.assert_not_called() - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--platform=platform.conf'] with pytest.raises(SystemExit) as excinfo: ips.main() assert excinfo.value.code == 2 - MockFramework.assert_not_called() + mock_framework.assert_not_called() - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--simulation=sim.cfg', '--platform=platform.conf'] ips.main() - MockFramework.assert_called_with(['sim.cfg'], 'sys.stdout', 'platform.conf', False, False, 0, 0) + mock_framework.assert_called_with( + ['sim.cfg'], 'sys.stdout', 'platform.conf', False, False, 0, 0 + ) os.environ['IPS_PLATFORM_FILE'] = 'platform.conf' - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--config=sim.cfg'] ips.main() - MockFramework.assert_called_with(['sim.cfg'], 'sys.stdout', 'platform.conf', False, False, 0, 0) + mock_framework.assert_called_with( + ['sim.cfg'], 'sys.stdout', 'platform.conf', False, False, 0, 0 + ) - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--config=sim1.cfg,sim2.cfg'] ips.main() - MockFramework.assert_called_with(['sim1.cfg', 'sim2.cfg'], 'sys.stdout', 'platform.conf', False, False, 0, 0) + mock_framework.assert_called_with( + ['sim1.cfg', 'sim2.cfg'], 'sys.stdout', 'platform.conf', False, False, 0, 0 + ) - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--config=sim.cfg', '--log=file.log'] ips.main() - MockFramework.assert_called_with(['sim.cfg'], 'file.log', 'platform.conf', False, False, 0, 0) + mock_framework.assert_called_with(['sim.cfg'], 'file.log', 'platform.conf', False, False, 0, 0) - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--config=sim.cfg', '--nodes=5', '--ppn=32'] ips.main() - MockFramework.assert_called_with(['sim.cfg'], 'sys.stdout', 'platform.conf', False, False, 5, 32) + mock_framework.assert_called_with( + ['sim.cfg'], 'sys.stdout', 'platform.conf', False, False, 5, 32 + ) - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--config=sim.cfg', '--debug'] ips.main() - MockFramework.assert_called_with(['sim.cfg'], 'sys.stdout', 'platform.conf', True, False, 0, 0) + mock_framework.assert_called_with(['sim.cfg'], 'sys.stdout', 'platform.conf', True, False, 0, 0) - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--config=sim.cfg', '--verbose'] ips.main() - MockFramework.assert_called_with(['sim.cfg'], 'sys.stdout', 'platform.conf', False, True, 0, 0) + mock_framework.assert_called_with(['sim.cfg'], 'sys.stdout', 'platform.conf', False, True, 0, 0) - MockFramework.reset_mock() + mock_framework.reset_mock() sys.argv = ['ips.py', '--config=sim.cfg', '--platform=workstation.conf'] ips.main() - MockFramework.assert_called_with(['sim.cfg'], 'sys.stdout', 'workstation.conf', False, False, 0, 0) + mock_framework.assert_called_with( + ['sim.cfg'], 'sys.stdout', 'workstation.conf', False, False, 0, 0 + ) diff --git a/tests/new/test_perlmutter_srun.py b/tests/new/test_perlmutter_srun.py index d590eb4e..52195bc6 100644 --- a/tests/new/test_perlmutter_srun.py +++ b/tests/new/test_perlmutter_srun.py @@ -38,7 +38,7 @@ def write_basic_config_and_platform_files(tmpdir): [DRIVER] CLASS = OPENMP SUB_CLASS = - NAME = gpu_task + NAME = GpuTask BIN_PATH = EXE = {tmpdir!s}/gpu_test.sh NPROC = 1 @@ -78,21 +78,31 @@ def test_srun_gpu_on_perlmutter(tmpdir): json_files = glob.glob(str(tmpdir.join('simulation_log').join('*.json'))) assert len(json_files) == 1 with open(json_files[0], 'r') as json_file: - comments = [json.loads(line)['comment'].split(', ', maxsplit=4)[3:] for line in json_file.readlines()] + comments = [json.loads(line)['comment'].split(', ', maxsplit=4)[3:] for line in json_file] - assert comments[5][0].startswith('Target = srun -N 1 -n 1 -c 64 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1') + assert comments[5][0].startswith( + 'Target = srun -N 1 -n 1 -c 64 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1' + ) assert comments[5][0].endswith('gpu_test.sh 1_1') - assert comments[7][0].startswith('Target = srun -N 1 -n 1 -c 64 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=2') + assert comments[7][0].startswith( + 'Target = srun -N 1 -n 1 -c 64 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=2' + ) assert comments[7][0].endswith('gpu_test.sh 1_2') - assert comments[9][0].startswith('Target = srun -N 1 -n 1 -c 64 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=4') + assert comments[9][0].startswith( + 'Target = srun -N 1 -n 1 -c 64 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=4' + ) assert comments[9][0].endswith('gpu_test.sh 1_4') - assert comments[11][0].startswith('Target = srun -N 1 -n 2 -c 32 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=2') + assert comments[11][0].startswith( + 'Target = srun -N 1 -n 2 -c 32 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=2' + ) assert comments[11][0].endswith('gpu_test.sh 2_2') - assert comments[13][0].startswith('Target = srun -N 1 -n 4 -c 16 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1') + assert comments[13][0].startswith( + 'Target = srun -N 1 -n 4 -c 16 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1' + ) assert comments[13][0].endswith('gpu_test.sh 4_1') # check that the process output log files are created diff --git a/tests/new/test_portal.py b/tests/new/test_portal.py index 6c542cfe..a02f8b4a 100644 --- a/tests/new/test_portal.py +++ b/tests/new/test_portal.py @@ -1,4 +1,5 @@ import hashlib +import importlib import json import sys from multiprocessing import Process, set_start_method @@ -51,7 +52,7 @@ def write_basic_config_and_platform_files(tmpdir): [DRIVER] CLASS = DRIVER SUB_CLASS = - NAME = driver + NAME = Driver BIN_PATH = NPROC = 1 INPUT_FILES = @@ -61,7 +62,7 @@ def write_basic_config_and_platform_files(tmpdir): [WORKER] CLASS = WORKER SUB_CLASS = - NAME = simple_sleep + NAME = SimpleSleep NPROC = 1 BIN_PATH = INPUT_FILES = @@ -78,18 +79,23 @@ def write_basic_config_and_platform_files(tmpdir): def test_portal(tmpdir): pytest.importorskip('flask') - from flask import Flask, jsonify, request # pylint: disable=import-outside-toplevel + flask = importlib.import_module('flask') + flask = flask.flask + jsonify = flask.jsonify + request = flask.request platform_file, config_file = write_basic_config_and_platform_files(tmpdir) # standup simple flask server to test send_post def flask_server(): - app = Flask('IPS portal') + app = flask('IPS portal') @app.route('/', methods=['POST']) def api(): data = request.get_json() - return jsonify(message='Events added to run', events=len(data), runid=42, event=data), 200 + return jsonify( + message='Events added to run', events=len(data), runid=42, event=data + ), 200 app.run(port=18080) @@ -112,14 +118,18 @@ def api(): with open(str(tmpdir.join('ips.log')), 'r') as f: lines = f.readlines() - URLs = [line[57:] for line in lines if 'FWK_COMP_PortalBridge_4 INFO' in line] - assert len(URLs) > 0 - assert URLs[0] == 'Run Portal URL = http://localhost:18080/42\n' + ur_ls = [line[57:] for line in lines if 'FWK_COMP_PortalBridge_4 INFO' in line] + assert len(ur_ls) > 0 + assert ur_ls[0] == 'Run Portal URL = http://localhost:18080/42\n' # remove timestamp and common start lines = [ (int(code), json.loads(data)) - for (code, data) in [line[74:].strip().split(maxsplit=1) for line in lines if 'FWK_COMP_PortalBridge_4 DEBUG Portal Response: ' in line] + for (code, data) in [ + line[74:].strip().split(maxsplit=1) + for line in lines + if 'FWK_COMP_PortalBridge_4 DEBUG Portal Response: ' in line + ] ] for code, _ in lines: @@ -160,7 +170,7 @@ def api(): assert 'duration' in trace assert 'timestamp' in trace assert 'id' in trace - assert trace['id'] == hashlib.md5('portal_test@FRAMEWORK@Framework@0'.encode()).hexdigest()[:16] + assert trace['id'] == hashlib.md5(b'portal_test@FRAMEWORK@Framework@0').hexdigest()[:16] assert 'traceId' in trace assert trace['traceId'] == hashlib.md5(event['portal_runid'].encode()).hexdigest() assert 'parentId' not in trace diff --git a/tests/new/test_resourceHelper.py b/tests/new/test_resourceHelper.py index bd821013..92c71a76 100644 --- a/tests/new/test_resourceHelper.py +++ b/tests/new/test_resourceHelper.py @@ -2,14 +2,14 @@ import pytest -from ipsframework.ipsExceptions import InvalidResourceSettingsException -from ipsframework.resourceHelper import getResourceList +from ipsframework.ipsExceptions import InvalidResourceSettingsError +from ipsframework.resource_helper import get_resource_list # checkjob @mock.patch('subprocess.Popen') -def test_resourceHelper_checkjob(subprocess_popen_mock, monkeypatch): +def test_resource_helper_checkjob(subprocess_popen_mock, monkeypatch): # mock the subprocess.Popen().returncode attribute and subprocess.Popen().stdout.readlines() type(subprocess_popen_mock.return_value).returncode = mock.PropertyMock(return_value=0) readlines = mock.Mock() @@ -35,26 +35,29 @@ def get_param(param, silent=True): monkeypatch.setenv('PBS_JOBID', '1234') # get resources from mock slurm env - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 2 - assert ['n27', '1'] in listOfNodes - assert ['n10', '4'] in listOfNodes + assert len(list_of_nodes) == 2 + assert ['n27', '1'] in list_of_nodes + assert ['n10', '4'] in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 4 - assert not accurateNodes + assert not accurate_nodes # qstat @mock.patch('subprocess.Popen') -def test_resourceHelper_qstat(subprocess_popen_mock, monkeypatch): +def test_resource_helper_qstat(subprocess_popen_mock, monkeypatch): # mock the subprocess.Popen().returncode attribute and subprocess.Popen().stdout.readlines() type(subprocess_popen_mock.return_value).returncode = mock.PropertyMock(return_value=0) readlines = mock.Mock() - readlines.readlines.return_value = [' Resource_List.mppwidth = 64 ', ' Resource_List.mppnppn = 2 '] + readlines.readlines.return_value = [ + ' Resource_List.mppwidth = 64 ', + ' Resource_List.mppnppn = 2 ', + ] type(subprocess_popen_mock.return_value).stdout = readlines # create mock services and get_platform_parameter return values @@ -68,46 +71,53 @@ def get_param(param, silent=True): # try with missing environment variables with pytest.raises(KeyError) as excinfo: - getResourceList(services, 'host') + get_resource_list(services, 'host') assert str(excinfo.value) == "'PBS_JOBID'" # set mock return values monkeypatch.setenv('PBS_JOBID', '1234') # get resources from mock slurm env - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 0 + assert len(list_of_nodes) == 0 assert cpn == 8 assert spn == 1 assert ppn == 2 - assert not accurateNodes + assert not accurate_nodes # now for HOST=stix - readlines.readlines.return_value = [' exec_host = compute1+compute2 ', ' Resource_List.nodect = 2 ', ' Resource_List.nodes = 2:ppn=2 '] + readlines.readlines.return_value = [ + ' exec_host = compute1+compute2 ', + ' Resource_List.nodect = 2 ', + ' Resource_List.nodes = 2:ppn=2 ', + ] monkeypatch.setenv('HOST', 'stix') # get resources from mock slurm env - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 2 - assert 'compute1' in listOfNodes - assert 'compute2' in listOfNodes + assert len(list_of_nodes) == 2 + assert 'compute1' in list_of_nodes + assert 'compute2' in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 2 - assert not accurateNodes + assert not accurate_nodes # qstat2 @mock.patch('subprocess.Popen') -def test_resourceHelper_qstat2(subprocess_popen_mock, monkeypatch): +def test_resource_helper_qstat2(subprocess_popen_mock, monkeypatch): # mock the subprocess.Popen().returncode attribute and subprocess.Popen().stdout.readlines() type(subprocess_popen_mock.return_value).returncode = mock.PropertyMock(return_value=0) readlines = mock.Mock() - readlines.readlines.return_value = [' exec_host = compute1/1+compute1/0+compute2/2+compute2/0 ', ' Hold_Types = n '] + readlines.readlines.return_value = [ + ' exec_host = compute1/1+compute1/0+compute2/2+compute2/0 ', + ' Hold_Types = n ', + ] type(subprocess_popen_mock.return_value).stdout = readlines # create mock services and get_platform_parameter return values @@ -121,28 +131,28 @@ def get_param(param, silent=True): # try with missing environment variables with pytest.raises(KeyError) as excinfo: - getResourceList(services, 'host') + get_resource_list(services, 'host') assert str(excinfo.value) == "'PBS_JOBID'" # set mock return values monkeypatch.setenv('PBS_JOBID', '1234') # get resources from mock slurm env - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 2 - assert ('compute1', ['1', '0']) in listOfNodes - assert ('compute2', ['2', '0']) in listOfNodes + assert len(list_of_nodes) == 2 + assert ('compute1', ['1', '0']) in list_of_nodes + assert ('compute2', ['2', '0']) in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 2 - assert accurateNodes + assert accurate_nodes # pbs_env -def test_resourceHelper_pbs_env(monkeypatch, tmpdir): +def test_resource_helper_pbs_env(monkeypatch, tmpdir): # create nodefile p = tmpdir.join('nodefile') p.write('compute0\ncompute1\n') @@ -157,41 +167,41 @@ def get_param(param, silent=True): # try with missing environment variables with pytest.raises(KeyError) as excinfo: - getResourceList(services, 'host') + get_resource_list(services, 'host') assert str(excinfo.value) == "'PBS_NNODES'" # PBS_NNODES monkeypatch.setenv('PBS_NNODES', '2') - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 2 - assert ('dummynode0', 1) in listOfNodes - assert ('dummynode1', 1) in listOfNodes + assert len(list_of_nodes) == 2 + assert ('dummynode0', 1) in list_of_nodes + assert ('dummynode1', 1) in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 1 - assert not accurateNodes + assert not accurate_nodes # PBS_NODEFILE monkeypatch.setenv('PBS_NODEFILE', str(p)) - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 2 - assert ('compute0', 1) in listOfNodes - assert ('compute1', 1) in listOfNodes + assert len(list_of_nodes) == 2 + assert ('compute0', 1) in list_of_nodes + assert ('compute1', 1) in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 1 - assert accurateNodes + assert accurate_nodes # slurm_env @mock.patch('subprocess.check_output') -def test_resourceHelper_slurm_env(subprocess_check_output_mock, monkeypatch): +def test_resource_helper_slurm_env(subprocess_check_output_mock, monkeypatch): subprocess_check_output_mock.return_value = 'nid00658\nnid00659\n' def get_param(param, silent=True): @@ -209,7 +219,7 @@ def get_param(param, silent=True): # try with missing environment variables with pytest.raises(KeyError) as excinfo: - getResourceList(services, 'host') + get_resource_list(services, 'host') assert str(excinfo.value) == "'SLURM_NODELIST'" # set mock return values @@ -217,73 +227,96 @@ def get_param(param, silent=True): # try with missing environment variables with pytest.raises(KeyError) as excinfo: - getResourceList(services, 'host') + get_resource_list(services, 'host') assert str(excinfo.value) == "'SLURM_JOB_TASKS_PER_NODE'" monkeypatch.setenv('SLURM_TASKS_PER_NODE', '2(x2)') # get resources from mock slurm env - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 2 - assert ('nid00658', 2) in listOfNodes - assert ('nid00659', 2) in listOfNodes + assert len(list_of_nodes) == 2 + assert ('nid00658', 2) in list_of_nodes + assert ('nid00659', 2) in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 2 - assert accurateNodes + assert accurate_nodes # manual -def test_resourceHelper_manual(): +def test_resource_helper_manual(): def get_param(param, silent=True): - params = {'CORES_PER_NODE': 8, 'SOCKETS_PER_NODE': 1, 'NODES': 2, 'PROCS_PER_NODE': 2, 'TOTAL_PROCS': 0, 'NODE_DETECTION': 'manual'} + params = { + 'CORES_PER_NODE': 8, + 'SOCKETS_PER_NODE': 1, + 'NODES': 2, + 'PROCS_PER_NODE': 2, + 'TOTAL_PROCS': 0, + 'NODE_DETECTION': 'manual', + } return params[param] # create mock services and get_platform_parameter return values services = mock.Mock() services.get_platform_parameter.side_effect = get_param - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 2 - assert ('dummynode0', 2) in listOfNodes - assert ('dummynode1', 2) in listOfNodes + assert len(list_of_nodes) == 2 + assert ('dummynode0', 2) in list_of_nodes + assert ('dummynode1', 2) in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 2 - assert not accurateNodes + assert not accurate_nodes -def test_resourceHelper_manual_InvalidException(): +def test_resource_helper_manual_invalid_exception(): # SOCKETS_PER_NODE > CORES_PER_NODE def get_param(param, silent=True): - params = {'CORES_PER_NODE': 8, 'SOCKETS_PER_NODE': 16, 'NODES': 2, 'PROCS_PER_NODE': 2, 'TOTAL_PROCS': 0, 'NODE_DETECTION': 'manual'} + params = { + 'CORES_PER_NODE': 8, + 'SOCKETS_PER_NODE': 16, + 'NODES': 2, + 'PROCS_PER_NODE': 2, + 'TOTAL_PROCS': 0, + 'NODE_DETECTION': 'manual', + } return params[param] # create mock services and get_platform_parameter return values services = mock.Mock() services.get_platform_parameter.side_effect = get_param - with pytest.raises(InvalidResourceSettingsException) as excinfo: - getResourceList(services, 'host') + with pytest.raises(InvalidResourceSettingsError) as excinfo: + get_resource_list(services, 'host') assert ( - str(excinfo.value) == 'Invalid resource specification in platform configuration file: socket per node count (16) greater than core per node count (8).' + str(excinfo.value) + == 'Invalid resource specification in platform configuration file: socket per node count (16) greater than core per node count (8).' ) # CORES_PER_NODE % SOCKETS_PER_NODE != 0 def get_param2(param, silent=True): - params = {'CORES_PER_NODE': 8, 'SOCKETS_PER_NODE': 3, 'NODES': 2, 'PROCS_PER_NODE': 2, 'TOTAL_PROCS': 0, 'NODE_DETECTION': 'manual'} + params = { + 'CORES_PER_NODE': 8, + 'SOCKETS_PER_NODE': 3, + 'NODES': 2, + 'PROCS_PER_NODE': 2, + 'TOTAL_PROCS': 0, + 'NODE_DETECTION': 'manual', + } return params[param] services.get_platform_parameter.side_effect = get_param2 - with pytest.raises(InvalidResourceSettingsException) as excinfo: - getResourceList(services, 'host') + with pytest.raises(InvalidResourceSettingsError) as excinfo: + get_resource_list(services, 'host') assert ( - str(excinfo.value) == 'Invalid resource specification in platform configuration file: socket per node count (3) ' + str(excinfo.value) + == 'Invalid resource specification in platform configuration file: socket per node count (3) ' 'not divisible by core per node count (8).' ) @@ -291,7 +324,7 @@ def get_param2(param, silent=True): # with no detection defined -def test_resourceHelper_no_detection(monkeypatch): +def test_resource_helper_no_detection(monkeypatch): # remove SLURM_NODELIST for tests if actually running with slurm monkeypatch.delenv('SLURM_NODELIST', raising=False) @@ -304,22 +337,29 @@ def get_param(param, silent=True): services.get_platform_parameter.side_effect = get_param with pytest.raises(KeyError) as excinfo: - getResourceList(services, 'host') + get_resource_list(services, 'host') assert str(excinfo.value) == "'NODES'" # fallback to manual is enough info supplied def get_param2(param, silent=True): - params = {'CORES_PER_NODE': 8, 'SOCKETS_PER_NODE': 1, 'NODES': 0, 'PROCS_PER_NODE': 0, 'TOTAL_PROCS': 0, 'NODE_DETECTION': ''} + params = { + 'CORES_PER_NODE': 8, + 'SOCKETS_PER_NODE': 1, + 'NODES': 0, + 'PROCS_PER_NODE': 0, + 'TOTAL_PROCS': 0, + 'NODE_DETECTION': '', + } return params[param] services.get_platform_parameter.side_effect = get_param2 - listOfNodes, cpn, spn, ppn, accurateNodes = getResourceList(services, 'host') + list_of_nodes, cpn, spn, ppn, accurate_nodes = get_resource_list(services, 'host') - assert len(listOfNodes) == 1 - assert ('dummynode0', 8) in listOfNodes + assert len(list_of_nodes) == 1 + assert ('dummynode0', 8) in list_of_nodes assert cpn == 8 assert spn == 1 assert ppn == 8 - assert not accurateNodes + assert not accurate_nodes diff --git a/tests/new/test_resourceManager.py b/tests/new/test_resourceManager.py index dc2a486a..3e9be7db 100644 --- a/tests/new/test_resourceManager.py +++ b/tests/new/test_resourceManager.py @@ -4,13 +4,13 @@ import pytest from ipsframework.ipsExceptions import ( - BadResourceRequestException, - GPUResourceRequestMismatchException, - InsufficientResourcesException, - ResourceRequestMismatchException, - ResourceRequestUnequalPartitioningException, + BadResourceRequestError, + GpuResourceRequestMismatchError, + InsufficientResourcesError, + ResourceRequestMismatchError, + ResourceRequestUnequalPartitioningError, ) -from ipsframework.resourceManager import ResourceManager +from ipsframework.resource_manager import ResourceManager def test_allocations(tmpdir): @@ -62,29 +62,41 @@ def test_allocations(tmpdir): # assert rm.check_core_cap(1, 16) == (False, 'insufficient') # assert rm.check_core_cap(4, 4) == (False, 'insufficient') - with pytest.raises(BadResourceRequestException) as excinfo: + with pytest.raises(BadResourceRequestError) as excinfo: rm.get_allocation(comp_id='comp0', nproc=12, task_id=0, whole_nodes=True, whole_socks=False) - assert str(excinfo.value) == 'component comp0 requested 3 nodes, which is more than possible by 1 nodes, for task 0.' + assert ( + str(excinfo.value) + == 'component comp0 requested 3 nodes, which is more than possible by 1 nodes, for task 0.' + ) - with pytest.raises(ResourceRequestUnequalPartitioningException) as excinfo: - rm.get_allocation(comp_id='comp0', nproc=3, task_id=0, whole_nodes=True, whole_socks=False, task_ppn=2) + with pytest.raises(ResourceRequestUnequalPartitioningError) as excinfo: + rm.get_allocation( + comp_id='comp0', nproc=3, task_id=0, whole_nodes=True, whole_socks=False, task_ppn=2 + ) assert ( - str(excinfo.value) == 'component comp0 requested 3 processes with 2 processes per node, while the number of processes requested is ' + str(excinfo.value) + == 'component comp0 requested 3 processes with 2 processes per node, while the number of processes requested is ' 'less than the max (8), it will result in unequal partitioning of processes across nodes' ) - with pytest.raises(BadResourceRequestException) as excinfo: + with pytest.raises(BadResourceRequestError) as excinfo: rm.get_allocation(comp_id='comp0', nproc=12, task_id=0, whole_nodes=False, whole_socks=True) - assert str(excinfo.value) == 'component comp0 requested 3 nodes, which is more than possible by 1 nodes, for task 0.' + assert ( + str(excinfo.value) + == 'component comp0 requested 3 nodes, which is more than possible by 1 nodes, for task 0.' + ) - with pytest.raises(ResourceRequestMismatchException) as excinfo: - rm.get_allocation(comp_id='comp0', nproc=6, task_id=0, whole_nodes=False, whole_socks=False, task_ppn=2) + with pytest.raises(ResourceRequestMismatchError) as excinfo: + rm.get_allocation( + comp_id='comp0', nproc=6, task_id=0, whole_nodes=False, whole_socks=False, task_ppn=2 + ) assert ( - str(excinfo.value) == 'component comp0 requested 6 processes with 2 processes per node, while the number of processes requested is ' + str(excinfo.value) + == 'component comp0 requested 6 processes with 2 processes per node, while the number of processes requested is ' 'less than the max (8), the processes per node value is too low.' ) @@ -172,10 +184,13 @@ def test_allocations(tmpdir): assert lines[7] == 'core: 2 - task_id: 1 - owner: comp0' assert lines[8] == 'core: 3 - task_id: 1 - owner: comp0' - with pytest.raises(InsufficientResourcesException) as excinfo: + with pytest.raises(InsufficientResourcesError) as excinfo: rm.get_allocation(comp_id='comp0', nproc=1, task_id=3, whole_nodes=False, whole_socks=False) - assert str(excinfo.value) == 'component comp0 requested 1 nodes, which is more than available by 0 nodes, for task 3.' + assert ( + str(excinfo.value) + == 'component comp0 requested 1 nodes, which is more than available by 0 nodes, for task 3.' + ) rm.release_allocation(task_id=1, status=None) @@ -290,27 +305,50 @@ def test_allocations(tmpdir): assert lines[8] == 'core: 3 - available' # test GPUs - with pytest.raises(GPUResourceRequestMismatchException) as excinfo: - rm.get_allocation(comp_id='comp0', nproc=1, task_gpp=1, task_id=0, whole_nodes=True, whole_socks=False) + with pytest.raises(GpuResourceRequestMismatchError) as excinfo: + rm.get_allocation( + comp_id='comp0', nproc=1, task_gpp=1, task_id=0, whole_nodes=True, whole_socks=False + ) - assert str(excinfo.value) == 'component comp0 requested 1 processes per node with 1 GPUs per process, which is greater than the available 0 GPUS_PER_NODE' + assert ( + str(excinfo.value) + == 'component comp0 requested 1 processes per node with 1 GPUs per process, which is greater than the available 0 GPUS_PER_NODE' + ) # set GPUS_PER_NODE to 2 rm = ResourceManager(fwk) rm.initialize(dm, tm, cm, cmd_nodes=2, cmd_ppn=4) rm.gpn = 2 - with pytest.raises(GPUResourceRequestMismatchException) as excinfo: - rm.get_allocation(comp_id='comp0', nproc=1, task_gpp=4, task_id=0, whole_nodes=True, whole_socks=False) + with pytest.raises(GpuResourceRequestMismatchError) as excinfo: + rm.get_allocation( + comp_id='comp0', nproc=1, task_gpp=4, task_id=0, whole_nodes=True, whole_socks=False + ) - assert str(excinfo.value) == 'component comp0 requested 1 processes per node with 4 GPUs per process, which is greater than the available 2 GPUS_PER_NODE' + assert ( + str(excinfo.value) + == 'component comp0 requested 1 processes per node with 4 GPUs per process, which is greater than the available 2 GPUS_PER_NODE' + ) - with pytest.raises(GPUResourceRequestMismatchException) as excinfo: - rm.get_allocation(comp_id='comp0', nproc=2, task_gpp=2, task_id=0, whole_nodes=True, whole_socks=False) + with pytest.raises(GpuResourceRequestMismatchError) as excinfo: + rm.get_allocation( + comp_id='comp0', nproc=2, task_gpp=2, task_id=0, whole_nodes=True, whole_socks=False + ) - assert str(excinfo.value) == 'component comp0 requested 2 processes per node with 2 GPUs per process, which is greater than the available 2 GPUS_PER_NODE' + assert ( + str(excinfo.value) + == 'component comp0 requested 2 processes per node with 2 GPUs per process, which is greater than the available 2 GPUS_PER_NODE' + ) - rm.get_allocation(comp_id='comp0', nproc=2, task_ppn=1, task_gpp=2, task_id=0, whole_nodes=True, whole_socks=False) + rm.get_allocation( + comp_id='comp0', + nproc=2, + task_ppn=1, + task_gpp=2, + task_id=0, + whole_nodes=True, + whole_socks=False, + ) with io.StringIO() as output: rm.nodes['dummy_node0'].print_sockets(output) diff --git a/tests/new/test_run_ensemble.py b/tests/new/test_run_ensemble.py index 1197f51e..e3b40f24 100644 --- a/tests/new/test_run_ensemble.py +++ b/tests/new/test_run_ensemble.py @@ -1,5 +1,6 @@ import logging import os +from importlib import import_module from ipsframework import ServicesProxy, TaskPool from ipsframework import services as services_module @@ -113,7 +114,9 @@ def record_submit_dask_tasks(*args): monkeypatch.setattr(TaskPool, 'distributed', object()) monkeypatch.setattr(task_pool, 'submit_dask_tasks', record_submit_dask_tasks) - assert task_pool.submit_tasks(use_dask=True, logfile='instance.out', errfile='instance.err') == 1 + assert ( + task_pool.submit_tasks(use_dask=True, logfile='instance.out', errfile='instance.err') == 1 + ) assert submitted_args[0][-2:] == ('instance.out', 'instance.err') @@ -183,9 +186,12 @@ def record_launch(executable, task_name, working_dir, *args, **kwargs): def test_launch_writes_stderr_to_logfile_when_errfile_is_omitted(tmpdir, monkeypatch): script = write_stdout_stderr_script(tmpdir) - import dask.distributed + dask_distributed = import_module('dask.distributed') + + def get_worker(): + return DummyDaskWorker() - monkeypatch.setattr(dask.distributed, 'get_worker', lambda: DummyDaskWorker()) + monkeypatch.setattr(dask_distributed, 'get_worker', get_worker) assert services_module.launch( str(script), @@ -203,9 +209,12 @@ def test_launch_writes_stderr_to_logfile_when_errfile_is_omitted(tmpdir, monkeyp def test_launch_writes_stderr_to_logfile_when_errfile_matches_logfile(tmpdir, monkeypatch): script = write_stdout_stderr_script(tmpdir) - import dask.distributed + dask_distributed = import_module('dask.distributed') + + def get_worker(): + return DummyDaskWorker() - monkeypatch.setattr(dask.distributed, 'get_worker', lambda: DummyDaskWorker()) + monkeypatch.setattr(dask_distributed, 'get_worker', get_worker) assert services_module.launch( str(script), diff --git a/tests/new/test_service_checkpoint_component.py b/tests/new/test_service_checkpoint_component.py index 4a783712..a78827b4 100644 --- a/tests/new/test_service_checkpoint_component.py +++ b/tests/new/test_service_checkpoint_component.py @@ -8,65 +8,74 @@ def test_checkpoint_components_bad_input(): # empty sim_conf sim_conf = {} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - servicesProxy.error = MagicMock(name='error') - servicesProxy.exception = MagicMock(name='exception') - servicesProxy._dispatch_checkpoint = MagicMock(name='dispatch_checkpoint') + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + services_proxy.error = MagicMock(name='error') + services_proxy.exception = MagicMock(name='exception') + services_proxy._dispatch_checkpoint = MagicMock(name='dispatch_checkpoint') with pytest.raises(KeyError) as excinfo: - servicesProxy.checkpoint_components([], 0) + services_proxy.checkpoint_components([], 0) assert str(excinfo.value) == "'CHECKPOINT'" - servicesProxy.error.assert_called_with('Missing CHECKPOINT config section, or one of the required parameters: MODE, NUM_CHECKPOINT') - servicesProxy.exception.assert_called_with('Error accessing CHECKPOINT section in config file') - servicesProxy._dispatch_checkpoint.assert_not_called() + services_proxy.error.assert_called_with( + 'Missing CHECKPOINT config section, or one of the required parameters: MODE, NUM_CHECKPOINT' + ) + services_proxy.exception.assert_called_with('Error accessing CHECKPOINT section in config file') + services_proxy._dispatch_checkpoint.assert_not_called() # missing NUM_CHECKPOINT - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'ALL'}} + services_proxy.sim_conf = {'CHECKPOINT': {'MODE': 'ALL'}} with pytest.raises(KeyError) as excinfo: - servicesProxy.checkpoint_components([], 0) + services_proxy.checkpoint_components([], 0) assert str(excinfo.value) == "'NUM_CHECKPOINT'" - servicesProxy.error.assert_called_with('Missing CHECKPOINT config section, or one of the required parameters: MODE, NUM_CHECKPOINT') - servicesProxy.exception.assert_called_with('Error accessing CHECKPOINT section in config file') - servicesProxy._dispatch_checkpoint.assert_not_called() + services_proxy.error.assert_called_with( + 'Missing CHECKPOINT config section, or one of the required parameters: MODE, NUM_CHECKPOINT' + ) + services_proxy.exception.assert_called_with('Error accessing CHECKPOINT section in config file') + services_proxy._dispatch_checkpoint.assert_not_called() # invalid MODE - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'NOT_A_MODE', 'NUM_CHECKPOINT': '-1'}} + services_proxy.sim_conf = {'CHECKPOINT': {'MODE': 'NOT_A_MODE', 'NUM_CHECKPOINT': '-1'}} with pytest.raises(Exception) as excinfo: - servicesProxy.checkpoint_components([], 0) + services_proxy.checkpoint_components([], 0) assert str(excinfo.value) == 'Invalid MODE = NOT_A_MODE in checkpoint configuration' - servicesProxy.error.assert_called_with('Invalid MODE = %s in checkpoint configuration', 'NOT_A_MODE') - servicesProxy._dispatch_checkpoint.assert_not_called() + services_proxy.error.assert_called_with( + 'Invalid MODE = %s in checkpoint configuration', 'NOT_A_MODE' + ) + services_proxy._dispatch_checkpoint.assert_not_called() def test_checkpoint_components_force(): # with Force=True, it should always call _dispatch_checkpoint - servicesProxy = ServicesProxy(None, None, None, {}, None) - servicesProxy._dispatch_checkpoint = MagicMock(name='dispatch_checkpoint') - servicesProxy.checkpoint_components([], 0, Force=True) - servicesProxy._dispatch_checkpoint.assert_called_once_with(0, [], False) + services_proxy = ServicesProxy(None, None, None, {}, None) + services_proxy._dispatch_checkpoint = MagicMock(name='dispatch_checkpoint') + services_proxy.checkpoint_components([], 0, Force=True) + services_proxy._dispatch_checkpoint.assert_called_once_with(0, [], False) def test_checkpoint_components_num_checkpoint(tmpdir): # NUM_CHECKPOINT=0, no checkpointing sim_conf = {'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '0'}, 'SIM_ROOT': '/some_dir'} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - servicesProxy.error = MagicMock(name='error') - servicesProxy.debug = MagicMock(name='debug') - servicesProxy._send_monitor_event = MagicMock(name='_send_monitor_event') - ret_dict = servicesProxy.checkpoint_components([], 0) + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + services_proxy.error = MagicMock(name='error') + services_proxy.debug = MagicMock(name='debug') + services_proxy._send_monitor_event = MagicMock(name='_send_monitor_event') + ret_dict = services_proxy.checkpoint_components([], 0) assert ret_dict is None # should be None since no checkpoint should happen - servicesProxy._send_monitor_event.assert_not_called() + services_proxy._send_monitor_event.assert_not_called() # NUM_CHECKPOINT=-1, checkpoint runs, keeping all checkpoints, no removing - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '-1'}, 'SIM_ROOT': '/some_dir'} - ret_dict = servicesProxy.checkpoint_components([], 0) + services_proxy.sim_conf = { + 'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '-1'}, + 'SIM_ROOT': '/some_dir', + } + ret_dict = services_proxy.checkpoint_components([], 0) assert ret_dict == {} # should be empty since no components - servicesProxy._send_monitor_event.assert_called_with('IPS_CHECKPOINT_START', 'Components = []') + services_proxy._send_monitor_event.assert_called_with('IPS_CHECKPOINT_START', 'Components = []') # NUM_CHECKPOINT=3, checkpoint runs, keeping only the most recent 3 checkpoints # create restart folder and add 10 timestamp checkpoints @@ -74,10 +83,13 @@ def test_checkpoint_components_num_checkpoint(tmpdir): for t in range(1, 11): restart_dir.mkdir(f'{t:.3f}') assert len(restart_dir.listdir()) == 10 - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '3'}, 'SIM_ROOT': str(tmpdir)} - ret_dict = servicesProxy.checkpoint_components([], 10) + services_proxy.sim_conf = { + 'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '3'}, + 'SIM_ROOT': str(tmpdir), + } + ret_dict = services_proxy.checkpoint_components([], 10) assert ret_dict == {} # should be empty since no components - servicesProxy._send_monitor_event.assert_called_with('IPS_CHECKPOINT_END', 'Components = []') + services_proxy._send_monitor_event.assert_called_with('IPS_CHECKPOINT_END', 'Components = []') # there should be only 3 remaining folder in restart (7, 8, 9) assert len(restart_dir.listdir()) == 3 @@ -89,14 +101,17 @@ def test_checkpoint_components_num_checkpoint(tmpdir): for t in range(11, 21): restart_dir.mkdir(f'{t:.3f}') assert len(restart_dir.listdir()) == 13 - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '3', 'PROTECT_FREQUENCY': '2'}, 'SIM_ROOT': str(tmpdir)} - servicesProxy.chkpt_counter = 19 - servicesProxy.new_chkpts = [f'{t:.3f}' for t in range(11, 21)] - servicesProxy.protected_chkpts = [f'{t:.3f}' for t in [12, 14, 16, 18]] - - ret_dict = servicesProxy.checkpoint_components([], 20) + services_proxy.sim_conf = { + 'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '3', 'PROTECT_FREQUENCY': '2'}, + 'SIM_ROOT': str(tmpdir), + } + services_proxy.chkpt_counter = 19 + services_proxy.new_chkpts = [f'{t:.3f}' for t in range(11, 21)] + services_proxy.protected_chkpts = [f'{t:.3f}' for t in [12, 14, 16, 18]] + + ret_dict = services_proxy.checkpoint_components([], 20) assert ret_dict == {} # should be empty since no components - servicesProxy._send_monitor_event.assert_called_with('IPS_CHECKPOINT_END', 'Components = []') + services_proxy._send_monitor_event.assert_called_with('IPS_CHECKPOINT_END', 'Components = []') # there should be every second from 12 and the 3 last non-protected checkpoints assert len(restart_dir.listdir()) == 8 @@ -108,80 +123,108 @@ def test_checkpoint_components_num_checkpoint(tmpdir): def test_checkpoint_components_modes(): # ALL sim_conf = {'CHECKPOINT': {'MODE': 'ALL', 'NUM_CHECKPOINT': '-1'}, 'SIM_ROOT': '/some_dir'} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - servicesProxy.error = MagicMock(name='error') - servicesProxy.debug = MagicMock(name='debug') - servicesProxy._dispatch_checkpoint = MagicMock(name='dispatch_checkpoint') - servicesProxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=5) - servicesProxy.start_time = 1000.0 - servicesProxy.last_ckpt_walltime = 1000.0 - servicesProxy.cur_time = 1010.0 - servicesProxy.checkpoint_components([], 0) - servicesProxy._dispatch_checkpoint.assert_called_once() + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + services_proxy.error = MagicMock(name='error') + services_proxy.debug = MagicMock(name='debug') + services_proxy._dispatch_checkpoint = MagicMock(name='dispatch_checkpoint') + services_proxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=5) + services_proxy.start_time = 1000.0 + services_proxy.last_ckpt_walltime = 1000.0 + services_proxy.cur_time = 1010.0 + services_proxy.checkpoint_components([], 0) + services_proxy._dispatch_checkpoint.assert_called_once() # WALLTIME_REGULAR - servicesProxy._dispatch_checkpoint.reset_mock() + services_proxy._dispatch_checkpoint.reset_mock() # 10 walltime interval, but only an interval of 2 has passed, shouldn't checkpoint - servicesProxy.start_time = 1000.0 - servicesProxy.last_ckpt_walltime = 1000.0 - servicesProxy.cur_time = 1002.0 - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'WALLTIME_REGULAR', 'WALLTIME_INTERVAL': '10', 'NUM_CHECKPOINT': '-1'}, 'SIM_ROOT': '/some_dir'} - servicesProxy.checkpoint_components([], 0) - servicesProxy._dispatch_checkpoint.assert_not_called() + services_proxy.start_time = 1000.0 + services_proxy.last_ckpt_walltime = 1000.0 + services_proxy.cur_time = 1002.0 + services_proxy.sim_conf = { + 'CHECKPOINT': { + 'MODE': 'WALLTIME_REGULAR', + 'WALLTIME_INTERVAL': '10', + 'NUM_CHECKPOINT': '-1', + }, + 'SIM_ROOT': '/some_dir', + } + services_proxy.checkpoint_components([], 0) + services_proxy._dispatch_checkpoint.assert_not_called() # 20 interval, so should call checkpoint - servicesProxy.cur_time = 1020.0 - servicesProxy.checkpoint_components([], 0) - servicesProxy._dispatch_checkpoint.assert_called_once() + services_proxy.cur_time = 1020.0 + services_proxy.checkpoint_components([], 0) + services_proxy._dispatch_checkpoint.assert_called_once() # WALLTIME_EXPLICIT - servicesProxy._dispatch_checkpoint.reset_mock() + services_proxy._dispatch_checkpoint.reset_mock() # 10 walltime interval, but only an interval of 2 has passed, shouldn't checkpoint - servicesProxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=2) - servicesProxy.start_time = 1000.0 - servicesProxy.last_ckpt_walltime = 1000.0 - servicesProxy.cur_time = 1002.0 - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'WALLTIME_EXPLICIT', 'WALLTIME_VALUES': '10 100', 'NUM_CHECKPOINT': '-1'}, 'SIM_ROOT': '/some_dir'} - servicesProxy.checkpoint_components([], 0) - servicesProxy._dispatch_checkpoint.assert_not_called() + services_proxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=2) + services_proxy.start_time = 1000.0 + services_proxy.last_ckpt_walltime = 1000.0 + services_proxy.cur_time = 1002.0 + services_proxy.sim_conf = { + 'CHECKPOINT': { + 'MODE': 'WALLTIME_EXPLICIT', + 'WALLTIME_VALUES': '10 100', + 'NUM_CHECKPOINT': '-1', + }, + 'SIM_ROOT': '/some_dir', + } + services_proxy.checkpoint_components([], 0) + services_proxy._dispatch_checkpoint.assert_not_called() # 20 elapsed time, so should call checkpoint - servicesProxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=20) - servicesProxy.cur_time = 1020.0 - servicesProxy.checkpoint_components([], 0) - servicesProxy._dispatch_checkpoint.assert_called_once() + services_proxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=20) + services_proxy.cur_time = 1020.0 + services_proxy.checkpoint_components([], 0) + services_proxy._dispatch_checkpoint.assert_called_once() # 50 elapsed time, should not call checkpoint since we have already checkpointed onve in this interval - servicesProxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=50) - servicesProxy.cur_time = 1050.0 - servicesProxy.last_ckpt_walltime = 1020.0 - servicesProxy.checkpoint_components([], 0) - servicesProxy._dispatch_checkpoint.assert_called_once() + services_proxy._get_elapsed_time = MagicMock(name='_get_elapsed_time', return_value=50) + services_proxy.cur_time = 1050.0 + services_proxy.last_ckpt_walltime = 1020.0 + services_proxy.checkpoint_components([], 0) + services_proxy._dispatch_checkpoint.assert_called_once() # PHYSTIME_REGULAR - servicesProxy.time_loop = [0, 10, 20, 30] - servicesProxy._dispatch_checkpoint.reset_mock() + services_proxy.time_loop = [0, 10, 20, 30] + services_proxy._dispatch_checkpoint.reset_mock() # physics time=10 interval=15, so should not call checkpoint - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'PHYSTIME_REGULAR', 'PHYSTIME_INTERVAL': '15', 'NUM_CHECKPOINT': '-1'}, 'SIM_ROOT': '/some_dir'} - assert servicesProxy.last_ckpt_phystime is None - servicesProxy.checkpoint_components([], 10) - assert servicesProxy.last_ckpt_phystime == 0 - servicesProxy._dispatch_checkpoint.assert_not_called() - - servicesProxy._dispatch_checkpoint.reset_mock() + services_proxy.sim_conf = { + 'CHECKPOINT': { + 'MODE': 'PHYSTIME_REGULAR', + 'PHYSTIME_INTERVAL': '15', + 'NUM_CHECKPOINT': '-1', + }, + 'SIM_ROOT': '/some_dir', + } + assert services_proxy.last_ckpt_phystime is None + services_proxy.checkpoint_components([], 10) + assert services_proxy.last_ckpt_phystime == 0 + services_proxy._dispatch_checkpoint.assert_not_called() + + services_proxy._dispatch_checkpoint.reset_mock() # physics time=20 interval=15, so should call checkpoint - servicesProxy.checkpoint_components([], 20) - servicesProxy._dispatch_checkpoint.assert_called_once() + services_proxy.checkpoint_components([], 20) + services_proxy._dispatch_checkpoint.assert_called_once() # PHYSTIME_EXPLICIT - servicesProxy._dispatch_checkpoint.reset_mock() - servicesProxy.sim_conf = {'CHECKPOINT': {'MODE': 'PHYSTIME_EXPLICIT', 'PHYSTIME_VALUES': '10 100', 'NUM_CHECKPOINT': '-1'}, 'SIM_ROOT': '/some_dir'} - servicesProxy.checkpoint_components([], 5) - servicesProxy._dispatch_checkpoint.assert_not_called() - - servicesProxy.checkpoint_components([], 20) - servicesProxy._dispatch_checkpoint.assert_called_once() - servicesProxy.last_ckpt_phystime = 20 - - servicesProxy.checkpoint_components([], 50) - servicesProxy._dispatch_checkpoint.assert_called_once() + services_proxy._dispatch_checkpoint.reset_mock() + services_proxy.sim_conf = { + 'CHECKPOINT': { + 'MODE': 'PHYSTIME_EXPLICIT', + 'PHYSTIME_VALUES': '10 100', + 'NUM_CHECKPOINT': '-1', + }, + 'SIM_ROOT': '/some_dir', + } + services_proxy.checkpoint_components([], 5) + services_proxy._dispatch_checkpoint.assert_not_called() + + services_proxy.checkpoint_components([], 20) + services_proxy._dispatch_checkpoint.assert_called_once() + services_proxy.last_ckpt_phystime = 20 + + services_proxy.checkpoint_components([], 50) + services_proxy._dispatch_checkpoint.assert_called_once() diff --git a/tests/new/test_taskManager.py b/tests/new/test_taskManager.py index 33b80e13..7bf5f3d6 100644 --- a/tests/new/test_taskManager.py +++ b/tests/new/test_taskManager.py @@ -5,15 +5,15 @@ from ipsframework import ResourceManager, TaskManager from ipsframework.ipsExceptions import ( - BadResourceRequestException, - BlockedMessageException, - GPUResourceRequestMismatchException, - InsufficientResourcesException, - ResourceRequestMismatchException, - ResourceRequestUnequalPartitioningException, + BadResourceRequestError, + BlockedMessageError, + GpuResourceRequestMismatchError, + InsufficientResourcesError, + ResourceRequestMismatchError, + ResourceRequestUnequalPartitioningError, ) from ipsframework.messages import ServiceRequestMessage -from ipsframework.taskManager import TaskInit +from ipsframework.task_manager import TaskInit def test_build_launch_cmd_fail(): @@ -32,7 +32,7 @@ def test_build_launch_cmd_fail(): ppn=None, max_ppn=None, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -46,7 +46,16 @@ def test_build_launch_cmd_eval(): tm.resource_mgr = mock.Mock(nodes=['node1']) cmd = tm.build_launch_cmd( - nproc=1, binary='executable', cmd_args=(), working_dir=None, ppn=None, max_ppn=None, nodes=None, accurateNodes=None, partial_nodes=None, task_id=None + nproc=1, + binary='executable', + cmd_args=(), + working_dir=None, + ppn=None, + max_ppn=None, + nodes=None, + accurate_nodes=None, + partial_nodes=None, + task_id=None, ) assert cmd == ('executable', None) @@ -59,7 +68,7 @@ def test_build_launch_cmd_eval(): ppn=None, max_ppn=None, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -80,7 +89,16 @@ def test_build_launch_cmd_mpirun(): tm.config_mgr.get_platform_parameter.return_value = 'OpenMPI-generic' cmd = tm.build_launch_cmd( - nproc=1, binary='executable', cmd_args=(), working_dir=None, ppn=None, max_ppn=None, nodes=None, accurateNodes=None, partial_nodes=None, task_id=None + nproc=1, + binary='executable', + cmd_args=(), + working_dir=None, + ppn=None, + max_ppn=None, + nodes=None, + accurate_nodes=None, + partial_nodes=None, + task_id=None, ) assert cmd == (f'{mpirun} -np 1 -x PYTHONPATH executable ', None) @@ -93,7 +111,7 @@ def test_build_launch_cmd_mpirun(): ppn=None, max_ppn=None, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, launch_cmd_extra_args='-extra 1', @@ -109,7 +127,7 @@ def test_build_launch_cmd_mpirun(): ppn=None, max_ppn=None, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -124,7 +142,7 @@ def test_build_launch_cmd_mpirun(): ppn=None, max_ppn=None, nodes='n1,n2', - accurateNodes=True, + accurate_nodes=True, partial_nodes=None, task_id=None, ) @@ -143,7 +161,7 @@ def test_build_launch_cmd_mpirun(): ppn=4, max_ppn=None, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -158,13 +176,16 @@ def test_build_launch_cmd_mpirun(): ppn=4, max_ppn=None, nodes='n1,n2', - accurateNodes=True, + accurate_nodes=True, partial_nodes=None, task_id=None, core_list=[('n1', ['0:1', '3:4']), ('n2', ['0:4'])], ) - assert cmd == ('mpirun n1 2 executable 13 42 : n2 1 executable 13 42', {'MPI_DSM_CPULIST': '1,16:4'}) + assert cmd == ( + 'mpirun n1 2 executable 13 42 : n2 1 executable 13 42', + {'MPI_DSM_CPULIST': '1,16:4'}, + ) def test_build_launch_cmd_mpiexec(): @@ -175,7 +196,16 @@ def test_build_launch_cmd_mpiexec(): tm.resource_mgr = mock.Mock(nodes=['node1']) cmd = tm.build_launch_cmd( - nproc=1, binary='executable', cmd_args=(), working_dir=None, ppn=None, max_ppn=None, nodes=None, accurateNodes=None, partial_nodes=None, task_id=None + nproc=1, + binary='executable', + cmd_args=(), + working_dir=None, + ppn=None, + max_ppn=None, + nodes=None, + accurate_nodes=None, + partial_nodes=None, + task_id=None, ) assert cmd == ('mpiexec -n 1 executable ', None) @@ -188,7 +218,7 @@ def test_build_launch_cmd_mpiexec(): ppn=None, max_ppn=None, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -206,7 +236,7 @@ def test_build_launch_cmd_mpiexec(): ppn=None, max_ppn=None, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -221,7 +251,7 @@ def test_build_launch_cmd_mpiexec(): ppn=None, max_ppn=None, nodes='n1,n2', - accurateNodes=True, + accurate_nodes=True, partial_nodes=None, task_id=None, ) @@ -245,7 +275,7 @@ def test_build_launch_cmd_aprun(): ppn=4, max_ppn=4, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -260,7 +290,7 @@ def test_build_launch_cmd_aprun(): ppn=4, max_ppn=4, nodes='n1,n2', - accurateNodes=True, + accurate_nodes=True, partial_nodes=None, task_id=None, ) @@ -277,7 +307,7 @@ def test_build_launch_cmd_aprun(): ppn=4, max_ppn=4, nodes=None, - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -292,7 +322,7 @@ def test_build_launch_cmd_aprun(): ppn=4, max_ppn=4, nodes='n1,n2', - accurateNodes=True, + accurate_nodes=True, partial_nodes=None, task_id=None, ) @@ -315,7 +345,7 @@ def test_build_launch_cmd_numactl(): ppn=None, max_ppn=None, nodes='n1,n2', - accurateNodes=None, + accurate_nodes=None, partial_nodes=None, task_id=None, ) @@ -330,7 +360,7 @@ def test_build_launch_cmd_numactl(): ppn=None, max_ppn=None, nodes='n1,n2', - accurateNodes=True, + accurate_nodes=True, partial_nodes=True, task_id=None, ) @@ -347,7 +377,16 @@ def test_build_launch_cmd_srun(): tm.resource_mgr.cores_per_node = 2 cmd = tm.build_launch_cmd( - nproc=4, binary='executable', cmd_args=(), working_dir=None, ppn=None, max_ppn=None, nodes='n1,n2', accurateNodes=None, partial_nodes=True, task_id=None + nproc=4, + binary='executable', + cmd_args=(), + working_dir=None, + ppn=None, + max_ppn=None, + nodes='n1,n2', + accurate_nodes=None, + partial_nodes=True, + task_id=None, ) assert cmd == ('srun -N 2 -n 4 executable ', None) @@ -360,7 +399,7 @@ def test_build_launch_cmd_srun(): ppn=None, max_ppn=None, nodes='n1,n2', - accurateNodes=None, + accurate_nodes=None, partial_nodes=True, task_id=None, ) @@ -375,7 +414,7 @@ def test_build_launch_cmd_srun(): ppn=None, max_ppn=None, nodes='n1,n2', - accurateNodes=None, + accurate_nodes=None, partial_nodes=True, task_id=None, launch_cmd_extra_args='-extra 1', @@ -391,7 +430,7 @@ def test_build_launch_cmd_srun(): ppn=2, max_ppn=None, nodes='n1,n2', - accurateNodes=None, + accurate_nodes=None, partial_nodes=False, task_id=None, cpp=1, @@ -408,14 +447,17 @@ def test_build_launch_cmd_srun(): ppn=1, max_ppn=None, nodes='n1,n2', - accurateNodes=None, + accurate_nodes=None, partial_nodes=False, task_id=None, cpp=2, omp=False, ) - assert cmd == ('srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores executable 13 42', None) + assert cmd == ( + 'srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores executable 13 42', + None, + ) cmd = tm.build_launch_cmd( nproc=4, @@ -425,7 +467,7 @@ def test_build_launch_cmd_srun(): ppn=2, max_ppn=None, nodes='n1,n2', - accurateNodes=None, + accurate_nodes=None, partial_nodes=False, task_id=None, cpp=1, @@ -445,7 +487,7 @@ def test_build_launch_cmd_srun(): ppn=1, max_ppn=None, nodes='n1,n2', - accurateNodes=None, + accurate_nodes=None, partial_nodes=False, task_id=None, cpp=2, @@ -467,7 +509,7 @@ def test_build_launch_cmd_srun(): ppn=None, max_ppn=None, nodes='n1', - accurateNodes=None, + accurate_nodes=None, partial_nodes=False, task_id=None, cpp=1, @@ -475,7 +517,10 @@ def test_build_launch_cmd_srun(): omp=False, ) - assert cmd == ('srun -N 1 -n 1 -c 1 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1 executable 13 42', None) + assert cmd == ( + 'srun -N 1 -n 1 -c 1 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1 executable 13 42', + None, + ) cmd = tm.build_launch_cmd( nproc=1, @@ -485,7 +530,7 @@ def test_build_launch_cmd_srun(): ppn=None, max_ppn=None, nodes='n1', - accurateNodes=None, + accurate_nodes=None, partial_nodes=False, task_id=None, cpp=1, @@ -493,7 +538,10 @@ def test_build_launch_cmd_srun(): omp=False, ) - assert cmd == ('srun -N 1 -n 1 -c 1 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=4 executable 13 42', None) + assert cmd == ( + 'srun -N 1 -n 1 -c 1 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=4 executable 13 42', + None, + ) def test_init_task_srun(tmpdir): @@ -515,11 +563,17 @@ def test_init_task_srun(tmpdir): rm.initialize(dm, tm, cm, cmd_nodes=2, cmd_ppn=2) tm.task_launch_cmd = 'srun' - rm.accurateNodes = True + rm.accurate_nodes = True def init_final_task(nproc, tppn, tcpt=0): task_id, cmd, _, cores_allocated = tm.init_task( - ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(nproc, 'exe', '/dir', tppn, tcpt, 0, True, True, True, False, [], None)) + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(nproc, 'exe', '/dir', tppn, tcpt, 0, True, True, True, False, [], None), + ) ) tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) return task_id, cmd, cores_allocated @@ -534,7 +588,7 @@ def init_final_task(nproc, tppn, tcpt=0): assert cmd == 'srun -N 1 -n 2 -c 1 --threads-per-core=1 --cpu-bind=cores exe ' assert cores == 2 - with pytest.raises(ResourceRequestUnequalPartitioningException): + with pytest.raises(ResourceRequestUnequalPartitioningError): init_final_task(3, 0) task_id, cmd, cores = init_final_task(4, 0) @@ -542,7 +596,7 @@ def init_final_task(nproc, tppn, tcpt=0): assert cmd == 'srun -N 2 -n 4 -c 1 --threads-per-core=1 --cpu-bind=cores exe ' assert cores == 4 - with pytest.raises(BadResourceRequestException): + with pytest.raises(BadResourceRequestError): init_final_task(5, 0) task_id, cmd, cores = init_final_task(1, 1) @@ -555,7 +609,7 @@ def init_final_task(nproc, tppn, tcpt=0): assert cmd == 'srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores exe ' assert cores == 4 - with pytest.raises(ResourceRequestMismatchException): + with pytest.raises(ResourceRequestMismatchError): init_final_task(3, 1) fwk.reset_mock() @@ -574,7 +628,9 @@ def init_final_task(nproc, tppn, tcpt=0): task_id, cmd, cores = init_final_task(1, 1, 4) assert task_id == 11 assert cmd == 'srun -N 1 -n 1 -c 2 --threads-per-core=1 --cpu-bind=cores exe ' - fwk.warning.assert_called_once_with('task cpp (4) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead') + fwk.warning.assert_called_once_with( + 'task cpp (4) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead' + ) fwk.reset_mock() task_id, cmd, cores = init_final_task(2, 1, 2) @@ -592,36 +648,79 @@ def init_final_task(nproc, tppn, tcpt=0): task_id, cmd, cores = init_final_task(2, 1, 12) assert task_id == 14 assert cmd == 'srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores exe ' - fwk.warning.assert_called_once_with('task cpp (12) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead') + fwk.warning.assert_called_once_with( + 'task cpp (12) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead' + ) ( task_id, cmd, _, _, - ) = tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(1, 'exe', '/dir', 0, 0, 0, True, True, True, False, [], '-extra 1'))) + ) = tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(1, 'exe', '/dir', 0, 0, 0, True, True, True, False, [], '-extra 1'), + ) + ) tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) assert task_id == 15 assert cmd == 'srun -N 1 -n 1 -c 2 --threads-per-core=1 --cpu-bind=cores -extra 1 exe ' # start two task, second should fail with Insufficient Resources depending on block task_id, cmd, _, _ = tm.init_task( - ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(4, 'exe', '/dir', 0, 0, 0, True, True, True, False, [], None)) + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(4, 'exe', '/dir', 0, 0, 0, True, True, True, False, [], None), + ) ) - with pytest.raises(BlockedMessageException): - tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(1, 'exe', '/dir', 0, 0, 0, True, True, True, False, [], None))) + with pytest.raises(BlockedMessageError): + tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(1, 'exe', '/dir', 0, 0, 0, True, True, True, False, [], None), + ) + ) - with pytest.raises(InsufficientResourcesException): - tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(1, 'exe', '/dir', 0, 0, 0, False, True, True, False, [], None))) + with pytest.raises(InsufficientResourcesError): + tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(1, 'exe', '/dir', 0, 0, 0, False, True, True, False, [], None), + ) + ) tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) # request GPUs when there are none - with pytest.raises(GPUResourceRequestMismatchException) as e: - tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(1, 'exe', '/dir', 0, 0, 1, False, True, True, False, [], None))) + with pytest.raises(GpuResourceRequestMismatchError) as e: + tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(1, 'exe', '/dir', 0, 0, 1, False, True, True, False, [], None), + ) + ) - assert str(e.value) == 'component id requested 1 processes per node with 1 GPUs per process, which is greater than the available 0 GPUS_PER_NODE' + assert ( + str(e.value) + == 'component id requested 1 processes per node with 1 GPUs per process, which is greater than the available 0 GPUS_PER_NODE' + ) # set GPUS_PER_NODE=2 rm.gpn = 2 @@ -630,7 +729,15 @@ def init_final_task(nproc, tppn, tcpt=0): cmd, _, _, - ) = tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(1, 'exe', '/dir', 0, 0, 1, False, True, True, False, [], None))) + ) = tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(1, 'exe', '/dir', 0, 0, 1, False, True, True, False, [], None), + ) + ) tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) assert task_id == 20 assert cmd == 'srun -N 1 -n 1 -c 2 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1 exe ' @@ -640,7 +747,15 @@ def init_final_task(nproc, tppn, tcpt=0): cmd, _, _, - ) = tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(2, 'exe', '/dir', 1, 0, 1, False, True, True, False, [], None))) + ) = tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(2, 'exe', '/dir', 1, 0, 1, False, True, True, False, [], None), + ) + ) tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) assert task_id == 21 assert cmd == 'srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1 exe ' @@ -650,7 +765,15 @@ def init_final_task(nproc, tppn, tcpt=0): cmd, _, _, - ) = tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(2, 'exe', '/dir', 1, 0, 2, False, True, True, False, [], None))) + ) = tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(2, 'exe', '/dir', 1, 0, 2, False, True, True, False, [], None), + ) + ) tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) assert task_id == 22 assert cmd == 'srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=2 exe ' @@ -660,15 +783,34 @@ def init_final_task(nproc, tppn, tcpt=0): cmd, _, _, - ) = tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(2, 'exe', '/dir', 2, 0, 1, False, True, True, False, [], None))) + ) = tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(2, 'exe', '/dir', 2, 0, 1, False, True, True, False, [], None), + ) + ) tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) assert task_id == 23 assert cmd == 'srun -N 1 -n 2 -c 1 --threads-per-core=1 --cpu-bind=cores --gpus-per-task=1 exe ' - with pytest.raises(GPUResourceRequestMismatchException) as e: - tm.init_task(ServiceRequestMessage('id', 'id', 'c', 'init_task', TaskInit(2, 'exe', '/dir', 2, 0, 2, False, True, True, False, [], None))) + with pytest.raises(GpuResourceRequestMismatchError) as e: + tm.init_task( + ServiceRequestMessage( + 'id', + 'id', + 'c', + 'init_task', + TaskInit(2, 'exe', '/dir', 2, 0, 2, False, True, True, False, [], None), + ) + ) - assert str(e.value) == 'component id requested 2 processes per node with 2 GPUs per process, which is greater than the available 2 GPUS_PER_NODE' + assert ( + str(e.value) + == 'component id requested 2 processes per node with 2 GPUs per process, which is greater than the available 2 GPUS_PER_NODE' + ) def test_init_task_pool_srun(tmpdir): @@ -690,11 +832,27 @@ def test_init_task_pool_srun(tmpdir): rm.initialize(dm, tm, cm, cmd_nodes=2, cmd_ppn=2) tm.task_launch_cmd = 'srun' - rm.accurateNodes = True + rm.accurate_nodes = True def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): if msg is None: - msg = {f'task{n}': TaskInit(nproc, f'exe{n}', '/dir', tppn, tcpp, 0, False, False, True, False, [f'arg{n}'], None) for n in range(number_of_tasks)} + msg = { + f'task{n}': TaskInit( + nproc, + f'exe{n}', + '/dir', + tppn, + tcpp, + 0, + False, + False, + True, + False, + [f'arg{n}'], + None, + ) + for n in range(number_of_tasks) + } retval = tm.init_task_pool(ServiceRequestMessage('id', 'id', 'c', 'init_task_pool', msg)) for task_id, _, _, _ in retval.values(): tm.finish_task(ServiceRequestMessage('id', 'id', 'c', 'finish_task', task_id, None)) @@ -714,7 +872,7 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): assert cmd == 'srun -N 1 -n 2 -c 1 --threads-per-core=1 --cpu-bind=cores exe0 arg0' assert cores == 2 - with pytest.raises(ResourceRequestUnequalPartitioningException): + with pytest.raises(ResourceRequestUnequalPartitioningError): init_final_task_pool(3, 0, 1) retval = init_final_task_pool(4, 0, 1) @@ -724,7 +882,7 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): assert cmd == 'srun -N 2 -n 4 -c 1 --threads-per-core=1 --cpu-bind=cores exe0 arg0' assert cores == 4 - with pytest.raises(BadResourceRequestException): + with pytest.raises(BadResourceRequestError): init_final_task_pool(5, 0, 1) retval = init_final_task_pool(1, 1, 1) @@ -741,7 +899,7 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): assert cmd == 'srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores exe0 arg0' assert cores == 4 - with pytest.raises(ResourceRequestMismatchException): + with pytest.raises(ResourceRequestMismatchError): init_final_task_pool(3, 1, 1) retval = init_final_task_pool(1, 0, 2) @@ -771,7 +929,13 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): assert cmd == 'srun -N 2 -n 4 -c 1 --threads-per-core=1 --cpu-bind=cores exe0 arg0' assert cores == 4 - retval = init_final_task_pool(msg={'task0': TaskInit(1, 'exe0', '/dir', 0, 0, 0, False, False, True, False, ('arg0',), '-extra 1')}) + retval = init_final_task_pool( + msg={ + 'task0': TaskInit( + 1, 'exe0', '/dir', 0, 0, 0, False, False, True, False, ('arg0',), '-extra 1' + ) + } + ) assert len(retval) == 1 task_id, cmd, _, cores = retval['task0'] assert task_id == 15 @@ -802,7 +966,9 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): task_id, cmd, _, cores = retval['task0'] assert task_id == 18 assert cmd == 'srun -N 1 -n 1 -c 2 --threads-per-core=1 --cpu-bind=cores exe0 arg0' - fwk.warning.assert_called_once_with('task cpp (4) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead') + fwk.warning.assert_called_once_with( + 'task cpp (4) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead' + ) fwk.reset_mock() retval = init_final_task_pool(2, 1, 1, 2) @@ -826,7 +992,9 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): task_id, cmd, _, cores = retval['task0'] assert task_id == 21 assert cmd == 'srun -N 2 -n 2 -c 2 --threads-per-core=1 --cpu-bind=cores exe0 arg0' - fwk.warning.assert_called_once_with('task cpp (4) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead') + fwk.warning.assert_called_once_with( + 'task cpp (4) exceeds maximum possible for 1 procs per node with 2 cores per node, using 2 cpus per proc instead' + ) # different size tasks msg = { @@ -849,7 +1017,7 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): 'task0': TaskInit(1, 'exe0', '/dir', 0, 0, 0, False, False, True, False, ('arg0',), None), 'task1': TaskInit(5, 'exe1', '/dir', 0, 0, 0, False, False, True, False, ('arg1',), None), } - with pytest.raises(BadResourceRequestException): + with pytest.raises(BadResourceRequestError): init_final_task_pool(msg=msg) # one good task, one bad task @@ -857,5 +1025,5 @@ def init_final_task_pool(nproc=1, tppn=0, number_of_tasks=1, tcpp=0, msg=None): 'task0': TaskInit(1, 'exe0', '/dir', 0, 0, 0, False, False, True, False, ('arg0',), None), 'task1': TaskInit(3, 'exe1', '/dir', 1, 0, 0, False, False, True, False, ('arg1',), None), } - with pytest.raises(ResourceRequestMismatchException): + with pytest.raises(ResourceRequestMismatchError): init_final_task_pool(msg=msg) diff --git a/tests/new/test_timeloop_checkpoint.py b/tests/new/test_timeloop_checkpoint.py index a247c013..ab2faa81 100644 --- a/tests/new/test_timeloop_checkpoint.py +++ b/tests/new/test_timeloop_checkpoint.py @@ -22,20 +22,20 @@ def write_basic_config_and_platform_files(tmpdir, restart=False): config_file = tmpdir.join('ips_restart.config') if restart else tmpdir.join('ips.config') - SIMULATION_MODE = 'RESTART' if restart else 'NORMAL' + simulation_mode = 'RESTART' if restart else 'NORMAL' sim_log = 'sim_restart.log' if restart else 'sim.log' - START = 162.5 if restart else 100 - FINISH = 200 if restart else 150 - NSTEP = 3 if restart else 4 + start = 162.5 if restart else 100 + finish = 200 if restart else 150 + nstep = 3 if restart else 4 config = f"""RUN_COMMENT = testing SIM_NAME = test LOG_FILE = {tmpdir!s}/{sim_log} LOG_LEVEL = INFO SIM_ROOT = {tmpdir!s} -SIMULATION_MODE = {SIMULATION_MODE} +simulation_mode = {simulation_mode} CURRENT_STATE = ${{SIM_NAME}}_ps.dat STATE_FILES = $CURRENT_STATE STATE_WORK_DIR = $SIM_ROOT/work/state @@ -52,7 +52,7 @@ def write_basic_config_and_platform_files(tmpdir, restart=False): [TIMELOOP_DRIVER] CLASS = TIMELOOP SUB_CLASS = - NAME = timeloop_driver + NAME = TimeloopDriver BIN_PATH = NPROC = 1 INPUT_FILES = @@ -62,7 +62,7 @@ def write_basic_config_and_platform_files(tmpdir, restart=False): [TIMELOOP_COMP] CLASS = TIMELOOP_COMP SUB_CLASS = - NAME = timeloop_comp + NAME = TimeloopComp BIN_PATH = NPROC = 1 INPUT_FILES = @@ -73,7 +73,7 @@ def write_basic_config_and_platform_files(tmpdir, restart=False): [TIMELOOP_COMP2] CLASS = TIMELOOP_COMP2 SUB_CLASS = - NAME = timeloop_comp + NAME = TimeloopComp BIN_PATH = NPROC = 1 INPUT_FILES = @@ -83,9 +83,9 @@ def write_basic_config_and_platform_files(tmpdir, restart=False): MODULE = components.workers.timeloop_comp [TIME_LOOP] MODE = REGULAR - START = {START} - FINISH = {FINISH} - NSTEP = {NSTEP} + start = {start} + finish = {finish} + nstep = {nstep} [CHECKPOINT] MODE = ALL NUM_CHECKPOINT = 2 @@ -122,7 +122,11 @@ def test_timeloop_checkpoint_restart(tmpdir): for time in ['100.0', '112.5', '125.0', '137.5', '150.0']: assert f'TIMELOOP_COMP__timeloop_comp_2 INFO step({time})\n' in lines assert f'TIMELOOP_COMP2__timeloop_comp_3 INFO step({time})\n' in lines - for comp in ['TIMELOOP__timeloop_driver_1', 'TIMELOOP_COMP__timeloop_comp_2', 'TIMELOOP_COMP2__timeloop_comp_3']: + for comp in [ + 'TIMELOOP__timeloop_driver_1', + 'TIMELOOP_COMP__timeloop_comp_2', + 'TIMELOOP_COMP2__timeloop_comp_3', + ]: assert f'{comp} INFO checkpoint({time})\n' in lines # check output files @@ -182,7 +186,7 @@ def test_timeloop_checkpoint_restart(tmpdir): assert results_dir.join('TIMELOOP_COMP2__timeloop_comp_3').join(f'w2_1_{time}.dat').exists() assert results_dir.join('TIMELOOP_COMP2__timeloop_comp_3').join(f'w2_2_{time}.dat').exists() - # Now do SIMULATION_MODE=RESTART + # Now do simulation_mode=RESTART platform_file, restart_config_file = write_basic_config_and_platform_files(tmpdir, restart=True) @@ -208,7 +212,11 @@ def test_timeloop_checkpoint_restart(tmpdir): for time in ['162.5', '175.0', '187.5', '200.0']: assert f'TIMELOOP_COMP__timeloop_comp_8 INFO step({time})\n' in lines assert f'TIMELOOP_COMP2__timeloop_comp_9 INFO step({time})\n' in lines - for comp in ['TIMELOOP__timeloop_driver_7', 'TIMELOOP_COMP__timeloop_comp_8', 'TIMELOOP_COMP2__timeloop_comp_9']: + for comp in [ + 'TIMELOOP__timeloop_driver_7', + 'TIMELOOP_COMP__timeloop_comp_8', + 'TIMELOOP_COMP2__timeloop_comp_9', + ]: assert f'{comp} INFO checkpoint({time})\n' in lines # check output files @@ -264,14 +272,21 @@ def test_timeloop_checkpoint_restart(tmpdir): assert work_files.join('TIMELOOP_COMP__timeloop_comp_8').join('test_ps.dat').exists() assert len(work_files.join('TIMELOOP_COMP__timeloop_comp_8').join('w1_1.dat').readlines()) == 11 assert len(work_files.join('TIMELOOP_COMP__timeloop_comp_8').join('w1_2.dat').readlines()) == 5 - assert len(work_files.join('TIMELOOP_COMP__timeloop_comp_8').join('test_ps.dat').readlines()) == 32 + assert ( + len(work_files.join('TIMELOOP_COMP__timeloop_comp_8').join('test_ps.dat').readlines()) == 32 + ) assert work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('w2_1.dat').exists() assert work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('w2_2.dat').exists() assert work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('test_ps.dat').exists() - assert len(work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('w2_1.dat').readlines()) == 11 + assert ( + len(work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('w2_1.dat').readlines()) == 11 + ) assert len(work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('w2_2.dat').readlines()) == 5 - assert len(work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('test_ps.dat').readlines()) == 33 + assert ( + len(work_files.join('TIMELOOP_COMP2__timeloop_comp_9').join('test_ps.dat').readlines()) + == 33 + ) # check output from services.stage_output_files @@ -285,35 +300,39 @@ def test_timeloop_checkpoint_restart(tmpdir): assert results_dir.join('TIMELOOP_COMP2__timeloop_comp_9').join(f'w2_2_{time}.dat').exists() -def test_TIME_LOOP(): - sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'START': '0', 'FINISH': '10', 'NSTEP': '10'}} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - tl = servicesProxy.get_time_loop() +def test_time_loop(): + sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'start': '0', 'finish': '10', 'nstep': '10'}} + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + tl = services_proxy.get_time_loop() assert tl == list(range(11)) - sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'START': '0 + 20 / 2', 'FINISH': '13 - 1', 'NSTEP': '2'}} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - tl = servicesProxy.get_time_loop() + sim_conf = { + 'TIME_LOOP': {'MODE': 'REGULAR', 'start': '0 + 20 / 2', 'finish': '13 - 1', 'nstep': '2'} + } + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + tl = services_proxy.get_time_loop() assert tl == [10, 11, 12] - sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'START': '10 * 2', 'FINISH': '10 ** 2', 'NSTEP': '2'}} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - tl = servicesProxy.get_time_loop() + sim_conf = { + 'TIME_LOOP': {'MODE': 'REGULAR', 'start': '10 * 2', 'finish': '10 ** 2', 'nstep': '2'} + } + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + tl = services_proxy.get_time_loop() assert tl == [20, 60, 100] - sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'START': '1e2', 'FINISH': '5e1', 'NSTEP': '2'}} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - tl = servicesProxy.get_time_loop() + sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'start': '1e2', 'finish': '5e1', 'nstep': '2'}} + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + tl = services_proxy.get_time_loop() assert tl == [100, 75, 50] sim_conf = {'TIME_LOOP': {'MODE': 'EXPLICIT', 'VALUES': '7 13 -42 1000'}} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - tl = servicesProxy.get_time_loop() + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + tl = services_proxy.get_time_loop() assert tl == [7, 13, -42, 1000] - sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'START': '1p2', 'FINISH': '10', 'NSTEP': '2'}} - servicesProxy = ServicesProxy(None, None, None, sim_conf, None) - servicesProxy.error = MagicMock(name='error') + sim_conf = {'TIME_LOOP': {'MODE': 'REGULAR', 'start': '1p2', 'finish': '10', 'nstep': '2'}} + services_proxy = ServicesProxy(None, None, None, sim_conf, None) + services_proxy.error = MagicMock(name='error') with pytest.raises(ValueError) as excinfo: - servicesProxy.get_time_loop() - assert str(excinfo.value) == 'Invalid TIME_LOOP value of START = 1p2' + services_proxy.get_time_loop() + assert str(excinfo.value) == 'Invalid TIME_LOOP value of start = 1p2' diff --git a/tests/new/test_trace.py b/tests/new/test_trace.py index 2b026884..a707c5dd 100644 --- a/tests/new/test_trace.py +++ b/tests/new/test_trace.py @@ -5,7 +5,9 @@ from ipsframework import Framework -def write_basic_config_and_platform_files(tmpdir, timeout='', logfile='', errfile='', nproc=1, exe='/bin/sleep', value='', shifter=False): +def write_basic_config_and_platform_files( + tmpdir, timeout='', logfile='', errfile='', nproc=1, exe='/bin/sleep', value='', shifter=False +): platform_file = tmpdir.join('platform.conf') platform = """MPIRUN = eval @@ -103,7 +105,16 @@ def test_trace_info(tmpdir): 'trace@FRAMEWORK@Framework@0', ] names = ['init(0)', '1', 'step(0)', '1', 'step(0)', 'step(0)', 'finalize(0)', None] - tags = [None, {'procs_requested': '1', 'cores_allocated': '1'}, {}, {'procs_requested': '1', 'cores_allocated': '1'}, {}, None, None, {'total_cores': '2'}] + tags = [ + None, + {'procs_requested': '1', 'cores_allocated': '1'}, + {}, + {'procs_requested': '1', 'cores_allocated': '1'}, + {}, + None, + None, + {'total_cores': '2'}, + ] parents = [7, 2, 5, 4, 5, 7, 7, None] for n, trace in enumerate(traces): @@ -116,12 +127,28 @@ def test_trace_info(tmpdir): if names[n]: assert trace['name'] == names[n] - assert trace['id'] == hashlib.md5(f'{trace["localEndpoint"]["serviceName"]}:{trace["name"]}:{call_ids[n]}'.encode()).hexdigest()[:16] + assert ( + trace['id'] + == hashlib.md5( + f'{trace["localEndpoint"]["serviceName"]}:{trace["name"]}:{call_ids[n]}'.encode() + ).hexdigest()[:16] + ) else: - assert trace['id'] == hashlib.md5(f'{trace["localEndpoint"]["serviceName"]}'.encode()).hexdigest()[:16] + assert ( + trace['id'] + == hashlib.md5(f'{trace["localEndpoint"]["serviceName"]}'.encode()).hexdigest()[:16] + ) if parents[n]: if names[parents[n]]: - assert trace['parentId'] == hashlib.md5(f'{service_names[parents[n]]}:{names[parents[n]]}:{call_ids[parents[n]]}'.encode()).hexdigest()[:16] + assert ( + trace['parentId'] + == hashlib.md5( + f'{service_names[parents[n]]}:{names[parents[n]]}:{call_ids[parents[n]]}'.encode() + ).hexdigest()[:16] + ) else: - assert trace['parentId'] == hashlib.md5(f'{service_names[parents[n]]}'.encode()).hexdigest()[:16] + assert ( + trace['parentId'] + == hashlib.md5(f'{service_names[parents[n]]}'.encode()).hexdigest()[:16] + ) diff --git a/tests/utils/test_ensemble_csv.py b/tests/utils/test_ensemble_csv.py index 3ab9bf73..bf9dea8c 100644 --- a/tests/utils/test_ensemble_csv.py +++ b/tests/utils/test_ensemble_csv.py @@ -6,8 +6,16 @@ def test_instances_to_csv(): with tempfile.NamedTemporaryFile() as tmp: variables = { - 'a_comp': {'A': [3, 2, 4], 'B': [2.34, 5.82, 0.1], 'C': ['"the quick, brown fox"', 'baz', 'quux']}, - 'another_comp': {'D': [7, 5, 9], 'B': [0.775, 0.08, 29.2], 'F': ['xyzzy', 'plud', 'thud']}, + 'a_comp': { + 'A': [3, 2, 4], + 'B': [2.34, 5.82, 0.1], + 'C': ['"the quick, brown fox"', 'baz', 'quux'], + }, + 'another_comp': { + 'D': [7, 5, 9], + 'B': [0.775, 0.08, 29.2], + 'F': ['xyzzy', 'plud', 'thud'], + }, } instances = group_ensemble_variables_into_instances(variables, 'this_is_my_name') expected_result = b'''\