From 784b07068223e5436d15caedcbbc57835a7657d4 Mon Sep 17 00:00:00 2001 From: Sophia Wen Date: Thu, 27 Aug 2026 11:27:32 -0400 Subject: [PATCH] Create temp_dir if missing and fail all ranks together instead of deadlocking _make_sbd_dir handed temp_dir straight to tempfile.mkdtemp, which does not create parent directories. A temp_dir (the --temp_dir option) pointing at a path that did not exist yet therefore raised FileNotFoundError on rank 0. That happened before the broadcast that hands the created directory to the other ranks, so the others blocked in the broadcast forever and the whole job hung with no output instead of erroring. Create temp_dir and any missing parents on rank 0 before mkdtemp, and wrap the setup so its outcome is broadcast: on failure every rank raises RuntimeError together rather than deadlocking. --- python/sbd_solver.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/python/sbd_solver.py b/python/sbd_solver.py index 3ab4c03..3e4f723 100644 --- a/python/sbd_solver.py +++ b/python/sbd_solver.py @@ -332,14 +332,32 @@ def _make_sbd_dir(mpi_comm, mpi_rank, temp_dir): regenerated fcidump.txt when ``fcidump_path`` is not provided). Returns (path, owns_sbd_dir) where owns_sbd_dir is True on rank 0 (so the caller knows to rmtree it on exit). + + ``temp_dir`` is created along with any missing parents, so a caller may + pass a path that does not exist yet (e.g. ``--temp_dir`` pointing at a + fresh scratch location). Creation happens only on rank 0; if it fails, the + error is broadcast so that every rank raises together, rather than the + other ranks deadlocking in the broadcast below while rank 0 unwinds. """ temp_dir = temp_dir or tempfile.gettempdir() + # (ok, payload): payload is the created directory when ok is True, else a + # message describing why rank 0 could not create it. if mpi_rank == 0: - sbd_dir_str = str(Path(tempfile.mkdtemp(prefix="sbd_files_", dir=temp_dir))) + try: + Path(temp_dir).mkdir(parents=True, exist_ok=True) + created = tempfile.mkdtemp(prefix="sbd_files_", dir=temp_dir) + result = (True, str(Path(created))) + except OSError as exc: + result = (False, f"{type(exc).__name__}: {exc}") else: - sbd_dir_str = None - sbd_dir_str = mpi_comm.bcast(sbd_dir_str, root=0) - return Path(sbd_dir_str), (mpi_rank == 0) + result = None + ok, payload = mpi_comm.bcast(result, root=0) + if not ok: + raise RuntimeError( + f"rank 0 could not create an SBD temporary directory under " + f"{temp_dir!r}: {payload}" + ) + return Path(payload), (mpi_rank == 0) def _load_or_regenerate_fcidump(