1. ipyparallel
  2. benchmarks
In [15]:
import os
from IPython.core.interactiveshell import InteractiveShell
from IPython.display import display, Markdown, SVG, HTML, IFrame
from profiling import profiling_code
from profiling.view_profiling_results import get_latest_results_dir, get_initial_results_dir
from profiling.profiling_code_runner import profiling_tasks
InteractiveShell.ast_node_interactivity = "all"

ipyparallel profiling results

Running profiling with py-spy

In [16]:
def get_results(dir_name, file_name):
    return os.path.join(dir_name, file_name)

def get_file_name(task_name, is_client):
    return f'{task_name}_{"client" if is_client else "scheduler"}.svg'

def get_latest_results(task_name, is_client=True):
    return get_results(get_latest_results_dir(), get_file_name(task_name, is_client))

def get_initial_results(task_name, is_client=True):
    return get_results(get_initial_results_dir(), get_file_name(task_name, is_client))

def display_svg(svg_file_path):
    display(IFrame(src=svg_file_path, width='100%', height=800))

Task: many_empty_tasks:

In [17]:
??profiling_code.profile_many_empty_tasks

Signature: profiling_code.profile_many_empty_tasks(lview, n, block=True)
Docstring: <no docstring>
Source:   
def profile_many_empty_tasks(lview, n, block=True):
    lview.map(echo(0), [None] * n, block=block)
File:      ~/ipyparallel_master_project/profiling/profiling_code.py
Type:      function

Client:

In [18]:
display_svg(get_latest_results('many_empty_tasks'))

Scheduler:

In [19]:
display_svg(get_latest_results('many_empty_tasks', is_client=False))

Task: many_empty_tasks_non_blocking:

In [20]:
??profiling_code.profile_many_empty_tasks

Signature: profiling_code.profile_many_empty_tasks(lview, n, block=True)
Docstring: <no docstring>
Source:   
def profile_many_empty_tasks(lview, n, block=True):
    lview.map(echo(0), [None] * n, block=block)
File:      ~/ipyparallel_master_project/profiling/profiling_code.py
Type:      function

Client:

In [21]:
display_svg(get_latest_results('many_empty_tasks_non_blocking'))

Scheduler:

In [22]:
display_svg(get_latest_results('many_empty_tasks_non_blocking', is_client=False))

Task: tasks_with_large_data

In [23]:
?? profiling_code.profile_tasks_with_large_data

Signature:  profiling_code.profile_tasks_with_large_data(lview, num_bytes)
Docstring: <no docstring>
Source:   
def profile_tasks_with_large_data(lview, num_bytes):
    for _ in range(100):
        for i in range(10):
            lview.apply_sync(echo(0), np.empty(num_bytes, dtype=np.int8))
File:      ~/ipyparallel_master_project/profiling/profiling_code.py
Type:      function

Client:

In [25]:
display_svg(get_latest_results('tasks_with_large_data'))

Scheduler:

In [26]:
display_svg(get_latest_results('tasks_with_large_data', is_client=False))

Task: echo_many_arguments:

In [27]:
??profiling_code.profile_echo_many_arguments

Signature: profiling_code.profile_echo_many_arguments(lview, number_of_arguments)
Docstring: <no docstring>
Source:   
def profile_echo_many_arguments(lview, number_of_arguments):
    lview.map(
        lambda x: echo_many_arguments(*x),
        [
            tuple(np.empty(1, dtype=np.int8) for n in range(number_of_arguments))
            for x in range(16)
        ],
        block=False,
    )
File:      ~/ipyparallel_master_project/profiling/profiling_code.py
Type:      function

Client:

In [28]:
display_svg(get_latest_results('echo_many_arguments'))

Scheduler:

In [29]:
display_svg(get_latest_results('echo_many_arguments', is_client=False))