Merge pull request #1401 from Libensemble/release/v_1.4.2

Release/v 1.4.2

Author: shuds13

.github/workflows/basic.yml

@@ -163,4 +163,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.23.4
+        - uses: crate-ci/typos@v1.23.6

.github/workflows/extra.yml

@@ -229,6 +229,12 @@ jobs:
             rm ./libensemble/tests/unit_tests/test_ufunc_runners.py
             rm ./libensemble/tests/unit_tests/test_executor_balsam.py
 
+        - name: Start Redis
+          if: matrix.os == 'ubuntu-latest'
+          uses: supercharge/redis-github-action@1.7.0
+          with:
+            redis-version: 7
+
         - name: Run extensive tests, Ubuntu
           if: matrix.os == 'ubuntu-latest'
           run: |
@@ -254,4 +260,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.23.4
+        - uses: crate-ci/typos@v1.23.6

.wci.yml

@@ -16,8 +16,8 @@ description: |
 language: Python
 
 release:
-  version: 1.4.1
-  date: 2024-07-29
+  version: 1.4.2
+  date: 2024-08-14
 
 documentation:
   general: https://libensemble.readthedocs.io

CHANGELOG.rst

@@ -8,6 +8,32 @@ GitHub issues are referenced, and can be viewed with hyperlinks on the `github r
 
 .. _`github releases page`: https://github.com/Libensemble/libensemble/releases
 
+Release 1.4.2
+--------------
+
+:Date: August 14, 2024
+
+* Fix under-utilized resource usage. #1398
+  * Fixes bug causing executor to wrongly increase processor counts when not all nodes are utilized.
+  * Fixes case where setting `num_gpus` to zero was treated as `None`.
+* Add missing PerlmutterGPU specs (these were detected anyway). #1393
+* Handle case where Perlmutter finds no partition. #1391
+* Launch environment scripts in shell. #1392
+
+:Examples:
+
+* Add proxystore example (uses a proxy in history array). #1326
+
+:Note:
+
+* Tests were run on Linux and MacOS with Python versions 3.9, 3.10, 3.11, 3.12
+* Heterogeneous workflows tested on Frontier (OLCF), Polaris (ALCF), and Perlmutter (NERSC).
+* Note that tests have been recently run on Aurora (ALCF), but the system was unavailable at time of release.
+
+:Known Issues:
+
+* See known issues section in the documentation.
+
 Release 1.4.1
 --------------
 
@@ -25,7 +51,6 @@ Release 1.4.1
 
 * See known issues section in the documentation.
 
-
 Release 1.4.0
 --------------
 

docs/requirements.txt

@@ -1,4 +1,4 @@
-sphinx<8
+sphinx<9
 sphinxcontrib-bibtex
 sphinxcontrib-spelling
 autodoc_pydantic

install/misc_feature_requirements.txt

@@ -1 +1,2 @@
-globus-compute-sdk==2.24.0
+globus-compute-sdk==2.25.0
+proxystore==0.7.0

install/testing_requirements.txt

@@ -1,9 +1,9 @@
-flake8==7.1.0
+flake8==7.1.1
 coverage==7.3.1
 pytest==8.3.2
 pytest-cov==5.0.0
 pytest-timeout==2.3.1
 mock==5.1.0
 python-dateutil==2.9.0.post0
 anyio==4.4.0
-matplotlib==3.9.1
+matplotlib==3.9.2

libensemble/ensemble.py

@@ -12,8 +12,8 @@
 from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs
 from libensemble.tools import add_unique_random_streams
 from libensemble.tools import parse_args as parse_args_f
-from libensemble.tools.parse_args import mpi_init
 from libensemble.tools import save_libE_output
+from libensemble.tools.parse_args import mpi_init
 from libensemble.utils.misc import specs_dump
 
 ATTR_ERR_MSG = 'Unable to load "{}". Is the function or submodule correctly named?'

libensemble/executors/mpi_executor.py

@@ -138,7 +138,7 @@ def set_resources(self, resources: Resources) -> None:
         self.resources = resources
 
     def _launch_with_retries(
-        self, task: Task, subgroup_launch: bool, wait_on_start: Union[bool, int], run_cmd: List[str]
+        self, task: Task, subgroup_launch: bool, wait_on_start: Union[bool, int], run_cmd: List[str], use_shell: bool
     ) -> None:
         """Launch task with retry mechanism"""
         retry_count = 0
@@ -156,6 +156,7 @@ def _launch_with_retries(
                         stdout=out,
                         stderr=err,
                         start_new_session=subgroup_launch,
+                        shell=use_shell,
                     )
             except Exception as e:
                 logger.warning(f"task {task.name} submit command failed on try {retry_count} with error {e}")
@@ -325,12 +326,9 @@ def submit(
         if not num_procs and not match_procs_to_gpus:
             num_procs = self.gen_nprocs
 
-        if not num_gpus:
+        if num_gpus is None:
             num_gpus = self.gen_ngpus
 
-        if not num_nodes and (self.gen_ngpus or self.gen_nprocs):
-            num_nodes = self.resources.worker_resources.local_node_count
-
         if mpi_runner_type is not None:
             if isinstance(mpi_runner_type, str):
                 mpi_config = {"mpi_runner": mpi_runner_type}
@@ -367,8 +365,10 @@ def submit(
 
         if env_script is not None:
             run_cmd = Executor._process_env_script(task, runline, env_script)
+            use_shell = True
         else:
             run_cmd = runline
+            use_shell = False
 
         if dry_run:
             logger.info(f"Test (No submit) Runline: {' '.join(run_cmd)}")
@@ -378,7 +378,7 @@ def submit(
             task._implement_env()
 
             # Launch Task
-            self._launch_with_retries(task, sglaunch, wait_on_start, run_cmd)
+            self._launch_with_retries(task, sglaunch, wait_on_start, run_cmd, use_shell)
 
             if not task.timer.timing and not task.finished:
                 task.timer.start()

libensemble/executors/mpi_runner.py

@@ -121,7 +121,7 @@ def _set_gpu_cli_option(self, wresources, extra_args, gpu_setting_name, gpu_valu
     def _set_gpu_env_var(self, wresources, task, gpus_per_node, gpus_env):
         """Add GPU environment variable setting to the tasks environment"""
         jassert(wresources.matching_slots, f"Cannot assign CPUs/GPUs to non-matching slots per node {wresources.slots}")
-        slot_list = wresources.get_slots_as_string(multiplier=wresources.gpus_per_rset, limit=gpus_per_node)
+        slot_list = wresources.get_slots_as_string(multiplier=wresources.gpus_per_rset_per_node, limit=gpus_per_node)
         task._add_to_env(gpus_env, slot_list)
 
     def _local_runner_set_gpus(self, task, wresources, extra_args, gpus_per_node, ppn):
@@ -171,7 +171,7 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args,
 
         # gpus per node for this worker.
         if wresources.doihave_gpus():
-            gpus_avail_per_node = wresources.slot_count * wresources.gpus_per_rset
+            gpus_avail_per_node = wresources.slot_count * wresources.gpus_per_rset_per_node
         else:
             gpus_avail_per_node = 0
 
@@ -224,6 +224,35 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args,
 
         return nprocs, nnodes, ppn, extra_args
 
+    def _get_min_nodes(self, nprocs, ppn, nnodes, ngpus, resources):
+        """Get minimum nodes needed to match configuration"""
+        if nnodes is not None:
+            return nnodes
+        if ppn:
+            return None  # nnodes gets processed later.
+        if resources is not None:
+            wresources = resources.worker_resources
+            total_nodes = wresources.local_node_count
+            procs_on_node = wresources.slot_count * wresources.procs_per_rset_per_node
+
+            if not nprocs and ngpus is None:
+                # Delay node evaluation to GPU assignment code
+                return None
+            proc_min_nodes = 1
+            gpu_min_nodes = 1
+            if nprocs:
+                proc_min_nodes = (nprocs + procs_on_node - 1) // procs_on_node
+            if ngpus:
+                gpus_on_node = wresources.slot_count * wresources.gpus_per_rset_per_node
+                gpu_min_nodes = (ngpus + gpus_on_node - 1) // gpus_on_node
+
+            min_nodes = max(proc_min_nodes, gpu_min_nodes)
+            nnodes = min(min_nodes, total_nodes)
+            # Must have atleast one processor per node to use GPUs
+            if nprocs:
+                nnodes = min(nnodes, nprocs)
+            return nnodes
+
     def _adjust_procs(self, nprocs, ppn, nnodes, ngpus, resources):
         """Adjust an invalid config"""
 
@@ -241,8 +270,8 @@ def adjust_resource(n_units, units_attr, units_name):
 
         if resources is not None:
             wresources = resources.worker_resources
-            ngpus = adjust_resource(ngpus, "gpus_per_rset", "ngpus")
-            nprocs = adjust_resource(nprocs, "procs_per_rset", "nprocs")
+            ngpus = adjust_resource(ngpus, "gpus_per_rset_per_node", "ngpus")
+            nprocs = adjust_resource(nprocs, "procs_per_rset_per_node", "nprocs")
         return nprocs, ngpus
 
     def get_mpi_specs(
@@ -284,6 +313,8 @@ def get_mpi_specs(
 
         if match_procs_to_gpus:
             jassert(no_config_set, "match_procs_to_gpus is mutually exclusive with either of nprocs/ppn")
+
+        nnodes = self._get_min_nodes(nprocs, ppn, nnodes, ngpus, resources)
         nprocs, ngpus = self._adjust_procs(nprocs, ppn, nnodes, ngpus, resources)
 
         if auto_assign_gpus or ngpus is not None:
@@ -294,7 +325,7 @@ def get_mpi_specs(
                 task, resources, nprocs, nnodes, ppn, ngpus, extra_args, match_procs_to_gpus
             )
 
-        rm_rpn = True if self.rm_rpn and ppn is None and nnodes is None else False
+        rm_rpn = self.rm_rpn and ppn is None and nnodes is None
 
         hostlist = None
         if machinefile and not self.mfile_support: