From b605e94e7872ac1b4db7151920094070364cc86e Mon Sep 17 00:00:00 2001 From: Bas Nijholt Date: Wed, 10 Jun 2026 14:41:25 -0700 Subject: [PATCH] Default to loky executor on all platforms Python 3.14 changed the default multiprocessing start method on Linux from "fork" to "forkserver" (python/cpython#84559), so workers of the stdlib ProcessPoolExecutor now re-import __main__ instead of inheriting it. Functions defined interactively (notebooks, executed doc pages) are pickled by reference and fail to unpickle in the workers with e.g.: AttributeError: module '__main__' has no attribute 'sphere' loky's reusable executor serializes functions by value via cloudpickle and was already the default on macOS and Windows (and loky is already a required dependency), so use it on Linux too. This also removes the now-dead lambda-pickling check in AsyncRunner that only applied when the default executor was ProcessPoolExecutor. --- adaptive/runner.py | 47 ++++++-------------- docs/source/tutorial/tutorial.parallelism.md | 3 +- 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/adaptive/runner.py b/adaptive/runner.py index 1ab058a16..9ee5a0aa8 100644 --- a/adaptive/runner.py +++ b/adaptive/runner.py @@ -6,8 +6,6 @@ import functools import inspect import itertools -import pickle -import platform import time import traceback import warnings @@ -44,16 +42,16 @@ # -- Runner definitions -if platform.system() == "Linux": - _default_executor = concurrent.ProcessPoolExecutor # type: ignore[misc] -else: - # On Windows and MacOS functions, the __main__ module must be - # importable by worker subprocesses. This means that - # ProcessPoolExecutor will not work in the interactive interpreter. - # On Linux the whole process is forked, so the issue does not appear. - # See https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor - # and https://github.com/python-adaptive/adaptive/issues/301 - _default_executor = loky.get_reusable_executor # type: ignore[misc] +# Functions submitted to a stdlib ProcessPoolExecutor must be importable from +# `__main__` by the worker, which fails for functions defined interactively +# (notebooks, doc pages). This used to work on Linux because workers were +# forked, but Python 3.14 changed the default start method on Linux to +# "forkserver", which re-imports `__main__` like Windows/macOS always did. +# loky serializes functions by value with cloudpickle, so it works everywhere. +# See https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor, +# https://github.com/python-adaptive/adaptive/issues/301, +# and https://github.com/python/cpython/issues/84559 +_default_executor = loky.get_reusable_executor class BaseRunner(metaclass=abc.ABCMeta): @@ -86,8 +84,7 @@ class BaseRunner(metaclass=abc.ABCMeta): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `loky.get_reusable_executor` is used. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -373,8 +370,7 @@ class BlockingRunner(BaseRunner): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `loky.get_reusable_executor` is used. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -520,8 +516,7 @@ class AsyncRunner(BaseRunner): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `loky.get_reusable_executor` is used. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -595,22 +590,6 @@ def __init__( retries: int = 0, raise_if_retries_exceeded: bool = True, ) -> None: - if ( - executor is None - and _default_executor is concurrent.ProcessPoolExecutor - and not inspect.iscoroutinefunction(learner.function) - ): - try: - pickle.dumps(learner.function) - except pickle.PicklingError as e: - raise ValueError( - "`learner.function` cannot be pickled (is it a lamdba function?)" - " and therefore does not work with the default executor." - " Either make sure the function is pickleble or use an executor" - " that might work with 'hard to pickle'-functions" - " , e.g. `ipyparallel` with `dill`." - ) from e - super().__init__( learner, goal=goal, diff --git a/docs/source/tutorial/tutorial.parallelism.md b/docs/source/tutorial/tutorial.parallelism.md index 5decc61d5..9ca43ddf2 100644 --- a/docs/source/tutorial/tutorial.parallelism.md +++ b/docs/source/tutorial/tutorial.parallelism.md @@ -16,7 +16,8 @@ Often you will want to evaluate the function on some remote computing resources. ## `concurrent.futures` -On Unix-like systems by default {class}`adaptive.Runner` creates a {class}`~concurrent.futures.ProcessPoolExecutor`, but you can also pass one explicitly e.g. to limit the number of workers: +By default {class}`adaptive.Runner` creates a `loky.get_reusable_executor`, which serializes functions by value so it also works with functions defined interactively (e.g. in a notebook). +You can also pass a {class}`~concurrent.futures.ProcessPoolExecutor` explicitly, e.g. to limit the number of workers, but then the function must be importable from `__main__` by the worker processes: ```python from concurrent.futures import ProcessPoolExecutor