Sharing Data during Multiprocessing

Here we show how data can be shared among multiple processes. Mind however, that this is conceptually a rather bad design since the single runs are no longer independent of each other. A better solution would be to simply return the data and sort it into a list during post-processing.

Download: example_12_sharing_data_between_processes.py

import multiprocessing as mp
import os  # For path names working under Windows and Linux

import numpy as np

from pypet import Environment, cartesian_product


def multiply(traj, result_list):
    """Example of a sophisticated simulation that involves multiplying two values.

    This time we will store tha value in a shared list and only in the end add the result.

    :param traj:

        Trajectory containing
        the parameters in a particular combination,
        it also serves as a container for results.


    """
    z = traj.x * traj.y
    result_list[traj.v_idx] = z


def main():
    # Create an environment that handles running
    filename = os.path.join("hdf5", "example_12.hdf5")
    env = Environment(
        trajectory="Multiplication",
        filename=filename,
        file_title="Example_12_Sharing_Data",
        overwrite_file=True,
        comment="The first example!",
        continuable=False,  # We have shared data in terms of a multiprocessing list,
        # so we CANNOT use the continue feature.
        multiproc=True,
        ncores=2,
    )

    # The environment has created a trajectory container for us
    traj = env.trajectory

    # Add both parameters
    traj.f_add_parameter("x", 1, comment="I am the first dimension!")
    traj.f_add_parameter("y", 1, comment="I am the second dimension!")

    # Explore the parameters with a cartesian product
    traj.f_explore(cartesian_product({"x": [1, 2, 3, 4], "y": [6, 7, 8]}))

    # We want a shared list where we can put all out results in. We use a manager for this:
    result_list = mp.Manager().list()
    # Let's make some space for potential results
    result_list[:] = [0 for _dummy in range(len(traj))]

    # Run the simulation
    env.run(multiply, result_list)

    # Now we want to store the final list as numpy array
    traj.f_add_result("z", np.array(result_list))

    # Finally let's print the result to see that it worked
    print(traj.z)

    # Disable logging and close all log-files
    env.disable_logging()


if __name__ == "__main__":
    main()