Sharing Data during MultiprocessingΒΆ
Here we show how data can be shared among multiple processes. Mind however, that this is conceptually a rather bad design since the single runs are no longer independent of each other. A better solution would be to simply return the data and sort it into a list during post-processing.
Download: example_12_sharing_data_between_processes.py
__author__ = 'Robert Meyer'
import multiprocessing as mp
import numpy as np
import os # For path names working under Windows and Linux
from pypet import Environment, cartesian_product
def multiply(traj, result_list):
"""Example of a sophisticated simulation that involves multiplying two values.
This time we will store tha value in a shared list and only in the end add the result.
:param traj:
Trajectory containing
the parameters in a particular combination,
it also serves as a container for results.
"""
z=traj.x*traj.y
result_list[traj.v_idx] = z
def main():
# Create an environment that handles running
filename = os.path.join('hdf5', 'example_12.hdf5')
env = Environment(trajectory='Multiplication',
filename=filename,
file_title='Example_12_Sharing_Data',
comment='The first example!',
continuable=False, # We have shared data in terms of a multiprocessing list,
# so we CANNOT use the continue feature.
multiproc=True,
ncores=2)
# The environment has created a trajectory container for us
traj = env.v_trajectory
# Add both parameters
traj.f_add_parameter('x', 1, comment='I am the first dimension!')
traj.f_add_parameter('y', 1, comment='I am the second dimension!')
# Explore the parameters with a cartesian product
traj.f_explore(cartesian_product({'x':[1,2,3,4], 'y':[6,7,8]}))
# We want a shared list where we can put all out results in. We use a manager for this:
result_list = mp.Manager().list()
# Let's make some space for potential results
result_list[:] =[0 for _dummy in range(len(traj))]
# Run the simulation
env.f_run(multiply, result_list)
# Now we want to store the final list as numpy array
traj.f_add_result('z', np.array(result_list))
# Finally let's print the result to see that it worked
print(traj.z)
#Disable logging and close all log-files
env.f_disable_logging()
if __name__ == '__main__':
main()