diff --git a/bundle/05_spleen_segmentation_lightning.ipynb b/bundle/05_spleen_segmentation_lightning.ipynb index 68af68eca..43cc966f8 100644 --- a/bundle/05_spleen_segmentation_lightning.ipynb +++ b/bundle/05_spleen_segmentation_lightning.ipynb @@ -38,7 +38,7 @@ "outputs": [], "source": [ "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"\n", - "!pip install -q pytorch-lightning~=2.0.0" + "!pip install -q pytorch-lightning>=2.1" ] }, { @@ -855,7 +855,7 @@ "execution_count": 11, "id": "c5ba337d-a5b0-47de-9ae2-1554a2cb4f86", "metadata": {}, - "outputs": [ + "outputs": [ { "name": "stdout", "output_type": "stream", diff --git a/competitions/MICCAI/surgtoolloc/preprocess_detect_scene_and_split_fold.ipynb b/competitions/MICCAI/surgtoolloc/preprocess_detect_scene_and_split_fold.ipynb index f83c5881b..16a55bcc3 100644 --- a/competitions/MICCAI/surgtoolloc/preprocess_detect_scene_and_split_fold.ipynb +++ b/competitions/MICCAI/surgtoolloc/preprocess_detect_scene_and_split_fold.ipynb @@ -398,7 +398,7 @@ ], "source": [ "for lb in labels:\n", - " print(f\"{lb:30} {df_scene[df_scene[lb]>0].scene.nunique()}\")" + " print(f\"{lb:30} {df_scene[df_scene[lb] > 0].scene.nunique()}\")" ] }, { diff --git a/deep_atlas/deep_atlas_tutorial.ipynb b/deep_atlas/deep_atlas_tutorial.ipynb index 580e2c345..799a85fd3 100644 --- a/deep_atlas/deep_atlas_tutorial.ipynb +++ b/deep_atlas/deep_atlas_tutorial.ipynb @@ -1255,7 +1255,7 @@ "val_interval = 5\n", "\n", "for epoch_number in range(max_epochs):\n", - " print(f\"Epoch {epoch_number+1}/{max_epochs}:\")\n", + " print(f\"Epoch {epoch_number + 1}/{max_epochs}:\")\n", "\n", " seg_net.train()\n", " losses = []\n", @@ -1797,7 +1797,7 @@ "best_reg_validation_loss = float(\"inf\")\n", "\n", "for epoch_number in range(max_epochs):\n", - " print(f\"Epoch {epoch_number+1}/{max_epochs}:\")\n", + " print(f\"Epoch {epoch_number + 1}/{max_epochs}:\")\n", "\n", " # ------------------------------------------------\n", " # reg_net training, with seg_net frozen\n", @@ -2320,7 +2320,7 @@ "preview_image(det, normalize_by=\"slice\", threshold=0)\n", "loss = lncc_loss(example_warped_image, img12[:, [0], :, :, :]).item()\n", "print(f\"Similarity loss: {loss}\")\n", - "print(f\"number of folds: {(det<=0).sum()}\")\n", + "print(f\"number of folds: {(det <= 0).sum()}\")\n", "\n", "del reg_net_example_output, img12, example_warped_image\n", "torch.cuda.empty_cache()" diff --git a/modules/interpretability/class_lung_lesion.ipynb b/modules/interpretability/class_lung_lesion.ipynb index f6afb0c8b..246ee4257 100644 --- a/modules/interpretability/class_lung_lesion.ipynb +++ b/modules/interpretability/class_lung_lesion.ipynb @@ -680,8 +680,8 @@ " name += \"lesion\" if label == 1 else \"non-lesion\"\n", " name += \"\\npred: \"\n", " name += \"lesion\" if pred_label == 1 else \"non-lesion\"\n", - " name += f\"\\nlesion: {y_pred[0,1]:.3}\"\n", - " name += f\"\\nnon-lesion: {y_pred[0,0]:.3}\"\n", + " name += f\"\\nlesion: {y_pred[0, 1]:.3}\"\n", + " name += f\"\\nnon-lesion: {y_pred[0, 0]:.3}\"\n", "\n", " # run CAM\n", " cam_result = cam(x=image, class_idx=None)\n", diff --git a/runner.sh b/runner.sh index 3296299fd..fc9c19ee8 100755 --- a/runner.sh +++ b/runner.sh @@ -84,6 +84,7 @@ doesnt_contain_max_epochs=("${doesnt_contain_max_epochs[@]}" maisi_inference_tut doesnt_contain_max_epochs=("${doesnt_contain_max_epochs[@]}" realism_diversity_metrics.ipynb) doesnt_contain_max_epochs=("${doesnt_contain_max_epochs[@]}" omniverse_integration.ipynb) doesnt_contain_max_epochs=("${doesnt_contain_max_epochs[@]}" hugging_face_pipeline_for_monai.ipynb) +doesnt_contain_max_epochs=("${doesnt_contain_max_epochs[@]}" msd_crossval_datalist_generator.ipynb) # inference/datalist-only notebook, no training loop # Execution of the notebook in these folders / with the filename cannot be automated skip_run_papermill=() @@ -135,6 +136,9 @@ skip_run_papermill=("${skip_run_papermill[@]}" .*learn2reg_oasis_unpaired_brain_ skip_run_papermill=("${skip_run_papermill[@]}" .*finetune_vista3d_for_hugging_face_pipeline.ipynb*) skip_run_papermill=("${skip_run_papermill[@]}" .*TCIA_PROSTATEx_Prostate_MRI_Anatomy_Model.ipynb*) # https://github.com/Project-MONAI/tutorials/issues/2029 skip_run_papermill=("${skip_run_papermill[@]}" .*maisi_inference_tutorial.ipynb*) +skip_run_papermill=("${skip_run_papermill[@]}" .*image_restoration.ipynb*) # monai.networks.nets.restormer not yet in dev branch +skip_run_papermill=("${skip_run_papermill[@]}" .*05_spleen_segmentation_lightning*) # requires GPU; hardcoded .to("cuda") with no CPU fallback +skip_run_papermill=("${skip_run_papermill[@]}" .*deep_atlas_tutorial*) # requires GPU; device hardcoded to "cuda:0" # output formatting separator="" @@ -160,6 +164,8 @@ autofix=false failfast=false pattern="" papermill_opt="" +jobs=1 +data_dir="" kernelspec="python3" @@ -169,7 +175,7 @@ NB_OUTPUT_LINE_CAP=100 function print_usage { echo "runner.sh [--no-run] [--no-checks] [--autofix] [-f/--failfast] [-p/--pattern ] [-h/--help]" - echo "[-v/--version] [--verbose]" + echo "[-v/--version] [--verbose] [-j/--jobs ] [--data-dir ]" echo "" echo "MONAI tutorials testing utilities. When running the notebooks, we first search for variables, such as" echo "\"max_epochs\" and set them to 1 to reduce testing time." @@ -180,12 +186,21 @@ function print_usage { echo " --autofix : autofix where possible" echo " --cell-standard : check guidelines standards such as ## setup environment cell blocks" echo " --copyright : check whether every source code and notebook has a copyright header" - echo " -f, --failfast : stop on first error" + echo " -f, --failfast : stop on first error (ignored when --jobs > 1)" echo " -p, --pattern : pattern of files to be run (added to \`find . -type f -name *.ipynb -and ! -wholename *.ipynb_checkpoints*\`)" echo " -h, --help : show this help message and exit" echo " -t, --test : shortcut to run a single notebook using pattern \`-and -wholename\`" echo " -v, --version : show MONAI and system version information and exit" - echo " --verbose : show papermill logs when testing the noteboobks" + echo " --verbose : show papermill logs when testing the notebooks" + echo " -j, --jobs N : run N notebooks in parallel (default: 1). Each notebook logs independently;" + echo " logs are printed in original order after all jobs finish." + echo " Note: parallel jobs share the same Python environment; use --jobs 1 when" + echo " notebooks do conflicting pip installs." + echo " --data-dir PATH : set MONAI_DATA_DIRECTORY to PATH before running notebooks. Notebooks that" + echo " respect this env var will persist downloads there instead of a temp dir." + echo " Tip: mount a host directory at this path in Docker to cache across runs:" + echo " docker run ... -v /host/data:/data -e MONAI_DATA_DIRECTORY=/data ..." + echo " or pass --data-dir /data to this script after the Docker bind-mount." echo "" echo "Examples:" echo "./runner.sh # run full tests (${green}recommended before making pull requests${noColor})." @@ -197,6 +212,8 @@ function print_usage { echo " # check filenames containing \"read\" or \"load\", but not if the" echo " whole path contains \"deepgrow\"." echo "./runner.sh --kernelspec \"kernel\" # Set the kernelspec value used to run notebooks, default is \"python3\"." + echo "./runner.sh -j 4 --data-dir /data/monai_cache" + echo " # run 4 notebooks in parallel; reuse cached downloads." echo "./runner.sh --no-checks --no-run --copyright echo " # test if all notebooks and scripts have the copyright header" echo "./runner.sh --no-checks --no-run --cell-standard @@ -247,6 +264,14 @@ do echo $pattern shift ;; + -j|--jobs) + jobs="$2" + shift + ;; + --data-dir) + data_dir="$2" + shift + ;; -k|--kernelspec) kernelspec="$2" shift @@ -429,6 +454,19 @@ fi base_path="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd "${base_path}" +# Export MONAI_DATA_DIRECTORY so notebooks persist downloads across runs. +# 109/117 runnable notebooks honour this env var (pattern: +# directory = os.environ.get("MONAI_DATA_DIRECTORY") +# root_dir = tempfile.mkdtemp() if directory is None else directory +# Without it each notebook re-downloads on every container run. +if [ -n "$data_dir" ]; then + mkdir -p "$data_dir" + export MONAI_DATA_DIRECTORY="$data_dir" + echo "Data cache: $data_dir (MONAI_DATA_DIRECTORY)" +elif [ -n "${MONAI_DATA_DIRECTORY:-}" ]; then + echo "Data cache: $MONAI_DATA_DIRECTORY (MONAI_DATA_DIRECTORY from environment)" +fi + function replace_text { oldString="${s}\s*=\s*[0-9]\+" newString="${s} = 1" @@ -481,24 +519,28 @@ fi ######################################################################## # # -# loop over files # +# per-notebook logic (used by both sequential and parallel paths) # # # ######################################################################## -for file in "${files[@]}"; do - current_test_successful=0 +# _run_notebook FILE RESULT_FILE +# Runs PEP8 checks and/or papermill for FILE. +# Writes 0 (pass) or 1 (fail) to RESULT_FILE. +# Must be called in a subshell so cwd changes are isolated. +function _run_notebook { + local file="$1" + local result_file="$2" + local current_test_successful=0 echo "${separator}${blue}Running $file${noColor}" - # Get to file's folder and get file contents + local path filename path="$(dirname "${file}")" filename="$(basename "${file}")" - cd ${base_path}/${path} + cd "${base_path}/${path}" - ######################################################################## - # # - # code checks # - # # - ######################################################################## + #################################################################### + # code checks # + #################################################################### if [ $doChecks = true ]; then if [ $autofix = true ]; then @@ -509,33 +551,26 @@ for file in "${files[@]}"; do --pipe "sed 's/ = list()/ = []/'" fi - # to check flake8, convert to python script, don't check - # magic cells, and don't check line length for comment - # lines (as this includes markdown), and then run flake8 echo Checking PEP8 compliance... jupytext "$filename" --opt custom_cell_magics="writefile" -w --to script -o - | \ sed 's/\(^\s*\)%/\1pass # %/' | \ sed 's/\(^#.*\)$/\1 # noqa: E501/' | \ flake8 - --show-source --extend-ignore=E203,N812,W503 --max-line-length 120 - success=$? - if [ ${success} -ne 0 ] - then + local success=$? + if [ ${success} -ne 0 ]; then print_error_msg "Try running with autofixes: ${green}--autofix${noColor}" - test_fail ${success} + current_test_successful=1 fi fi - ######################################################################## - # # - # run notebooks with papermill # - # # - ######################################################################## - if [ $doRun = true ]; then - - skipRun=false + #################################################################### + # run notebook with papermill # + #################################################################### + if [ $doRun = true ] && [ $current_test_successful -eq 0 ]; then + local skipRun=false for skip_pattern in "${skip_run_papermill[@]}"; do - if [[ $file =~ $skip_pattern ]]; then + if [[ $file =~ $skip_pattern ]]; then echo "Skip Pattern Match" skipRun=true break @@ -544,45 +579,112 @@ for file in "${files[@]}"; do if [ $skipRun = true ]; then echo "Skipping" - continue + echo "$current_test_successful" > "$result_file" + return fi echo Running notebook... + local notebook notebook=$(cat "$filename") - # if compulsory keyword, max_epochs, missing... if [[ ! "$notebook" =~ "max_epochs" ]]; then - # and notebook isn't in list of those expected to not have that keyword... - should_contain_max_epochs=true + local should_contain_max_epochs=true for e in "${doesnt_contain_max_epochs[@]}"; do [[ "$e" == "$filename" ]] && should_contain_max_epochs=false && break done - # then error if [[ $should_contain_max_epochs == true ]]; then print_error_msg "Couldn't find the keyword \"max_epochs\", and the notebook wasn't on the list of expected exemptions (\"doesnt_contain_max_epochs\")." - test_fail 1 + current_test_successful=1 + echo "$current_test_successful" > "$result_file" + return fi fi - # Set some variables to 1 to speed up proceedings - strings_to_replace=(max_epochs val_interval disc_train_interval disc_train_steps num_batches_for_histogram) + local strings_to_replace=(max_epochs val_interval disc_train_interval disc_train_steps num_batches_for_histogram) for s in "${strings_to_replace[@]}"; do replace_text done python -c 'import monai; monai.config.print_config()' + local cmd cmd=$(echo "papermill ${papermill_opt} --progress-bar --log-output -k ${kernelspec}") echo "$cmd" + local out time out=$(echo "$notebook" | eval "$cmd") - success=$? + local success=$? if [[ ${success} -ne 0 || "$out" =~ "\"status\": \"failed\"" ]]; then - test_fail ${success} + current_test_successful=1 fi fi - num_tested=$((num_tested + 1)) - if [[ ${current_test_successful} -eq 0 ]]; then - num_successful_tests=$((num_successful_tests + 1)) - fi -done + echo "$current_test_successful" > "$result_file" +} + +######################################################################## +# # +# loop over files — sequential (jobs=1) or parallel (jobs>1) # +# # +######################################################################## +if [ "$jobs" -le 1 ]; then + # ---------------------------------------------------------------- # + # Sequential path — original behaviour, unchanged # + # ---------------------------------------------------------------- # + for file in "${files[@]}"; do + current_test_successful=0 + _result_file=$(mktemp) + + ( trap - EXIT; _run_notebook "$file" "$_result_file" ) + + current_test_successful=$(cat "$_result_file" 2>/dev/null || echo 1) + rm -f "$_result_file" + + num_tested=$((num_tested + 1)) + if [[ ${current_test_successful} -eq 0 ]]; then + num_successful_tests=$((num_successful_tests + 1)) + elif [ $failfast = true ]; then + finish + fi + done + +else + # ---------------------------------------------------------------- # + # Parallel path — N notebooks run concurrently # + # ---------------------------------------------------------------- # + echo "Running ${#files[@]} notebooks with --jobs $jobs" + _work_dir=$(mktemp -d) + + for file in "${files[@]}"; do + # Throttle: wait until a slot is free + while [ "$(jobs -rp | wc -l)" -ge "$jobs" ]; do + wait -n 2>/dev/null || sleep 0.2 + done + + _slug=$(printf '%s' "$file" | tr '/.' '--') + _log="${_work_dir}/${_slug}.log" + _result="${_work_dir}/${_slug}.result" + + ( + trap - EXIT + set +e + _run_notebook "$file" "$_result" + ) > "$_log" 2>&1 & + done + + wait # wait for all remaining background jobs + + # Print logs in original notebook order; collect pass/fail counts + for file in "${files[@]}"; do + _slug=$(printf '%s' "$file" | tr '/.' '--') + _log="${_work_dir}/${_slug}.log" + _result="${_work_dir}/${_slug}.result" + + cat "$_log" + num_tested=$((num_tested + 1)) + if [ "$(cat "$_result" 2>/dev/null)" = "0" ]; then + num_successful_tests=$((num_successful_tests + 1)) + fi + done + + rm -rf "$_work_dir" +fi