Fix lingering output files in test_symlinking and test_output_files

- Enhanced symlink_output_files to auto-create parent directories
- Fixed path handling for entities with sub-entities (Orchestrator/Ensemble)
- Ensured all tests use proper test directories instead of repo root
- Removed unused CONFIG imports
- All tests now pass without creating lingering files in repo root

Author: Al Rigazzi

smartsim/_core/control/controller.py

@@ -373,6 +373,10 @@ def symlink_output_files(
         historical_err.touch()
         historical_out.touch()
 
+        # Ensure the entity directory exists for symlinks
+        entity_out.parent.mkdir(parents=True, exist_ok=True)
+        entity_err.parent.mkdir(parents=True, exist_ok=True)
+
         if historical_err.exists() and historical_out.exists():
             entity_out.symlink_to(historical_out)
             entity_err.symlink_to(historical_err)

tests/test_output_files.py

@@ -30,7 +30,6 @@
 import pytest
 
 from smartsim import Experiment
-from smartsim._core.config import CONFIG
 from smartsim._core.control.controller import Controller, _AnonymousBatchJob
 from smartsim._core.launcher.step import Step
 from smartsim.database.orchestrator import Orchestrator
@@ -106,35 +105,45 @@ def test_mutated_model_output(test_dir):
 def test_get_output_files_with_create_job_step(test_dir):
     """Testing output files through _create_job_step"""
     exp_dir = pathlib.Path(test_dir)
-    status_dir = exp_dir / ".smartsim"
-    # Set the model path to the test directory
-    model.path = test_dir
-    step = controller._create_job_step(model)
-    expected_out_path = status_dir / (model.name + ".out")
-    expected_err_path = status_dir / (model.name + ".err")
+    # Create a fresh model instance for this test
+    test_model = Model("test_model", params={}, path=test_dir, run_settings=rs)
+    # Create run_dir to avoid using current working directory
+    run_dir = exp_dir / ".smartsim" / "run_test"
+    step = controller._create_job_step(test_model, run_dir)
+    expected_out_path = run_dir / (test_model.name + ".out")
+    expected_err_path = run_dir / (test_model.name + ".err")
     assert step.get_output_files() == (str(expected_out_path), str(expected_err_path))
 
 
 @pytest.mark.parametrize(
-    "entity",
-    [pytest.param(ens, id="ensemble"), pytest.param(orc, id="orchestrator")],
+    "entity_type",
+    [
+        pytest.param("ensemble", id="ensemble"),
+        pytest.param("orchestrator", id="orchestrator"),
+    ],
 )
-def test_get_output_files_with_create_batch_job_step(entity, test_dir):
+def test_get_output_files_with_create_batch_job_step(entity_type, test_dir):
     """Testing output files through _create_batch_job_step"""
     exp_dir = pathlib.Path(test_dir)
-    # Set the entity path to test_dir
-    entity.path = test_dir
-    batch_step, substeps = slurm_controller._create_batch_job_step(entity)
-    for step in substeps:
-        # With the new simplified structure, each step should use its own entity's path
-        # Each entity member has their own individual path, so the output goes in their own .smartsim directory
-        step_entity_path = pathlib.Path(step.meta["status_dir"]).parent
-        expected_out_path = pathlib.Path(step.meta["status_dir"]) / (
-            step.entity_name + ".out"
+
+    # Create fresh entities for each test to avoid path conflicts
+    if entity_type == "ensemble":
+        entity = Ensemble(
+            "ens", params={}, run_settings=rs, batch_settings=bs, replicas=3
         )
-        expected_err_path = pathlib.Path(step.meta["status_dir"]) / (
-            step.entity_name + ".err"
+    else:  # orchestrator
+        entity = Orchestrator(
+            db_nodes=3, batch=True, launcher="slurm", run_command="srun"
         )
+
+    entity.path = test_dir
+    # Create run_dir to avoid using current working directory
+    run_dir = exp_dir / ".smartsim" / "run_test_batch"
+    batch_step, substeps = slurm_controller._create_batch_job_step(entity, run_dir)
+    for step in substeps:
+        # With timestamped runs, output files should be in the run_dir
+        expected_out_path = run_dir / (step.entity_name + ".out")
+        expected_err_path = run_dir / (step.entity_name + ".err")
         actual_out, actual_err = step.get_output_files()
         assert actual_out == str(expected_out_path)
         assert actual_err == str(expected_err_path)

tests/test_symlinking.py

@@ -30,7 +30,6 @@
 import pytest
 
 from smartsim import Experiment
-from smartsim._core.config import CONFIG
 from smartsim._core.control.controller import Controller, _AnonymousBatchJob
 from smartsim.database.orchestrator import Orchestrator
 from smartsim.entity.ensemble import Ensemble
@@ -58,50 +57,84 @@
 
 
 @pytest.mark.parametrize(
-    "entity",
-    [pytest.param(ens, id="ensemble"), pytest.param(model, id="model")],
+    "entity_type",
+    [pytest.param("ensemble", id="ensemble"), pytest.param("model", id="model")],
 )
-def test_symlink(test_dir, entity):
+def test_symlink(test_dir, entity_type):
     """Test symlinking historical output files"""
-    entity.path = test_dir
-    if entity.type == Ensemble:
-        for member in ens.models:
+    if entity_type == "ensemble":
+        entity = Ensemble(
+            "ens", params={}, run_settings=rs, batch_settings=bs, replicas=3
+        )
+        entity.path = test_dir
+        for member in entity.models:
             symlink_with_create_job_step(test_dir, member)
     else:
+        entity = Model("test_model", params={}, path=test_dir, run_settings=rs)
         symlink_with_create_job_step(test_dir, entity)
 
 
 def symlink_with_create_job_step(test_dir, entity):
     """Function that helps cut down on repeated testing code"""
     exp_dir = pathlib.Path(test_dir)
     entity.path = test_dir
-    # With simplified structure, output files go directly in .smartsim directory
-    status_dir = exp_dir / ".smartsim"
-    step = controller._create_job_step(entity)
+    # Create run_dir to simulate timestamped run structure
+    run_dir = exp_dir / ".smartsim" / "run_test"
+    step = controller._create_job_step(entity, run_dir)
     controller.symlink_output_files(step, entity)
     assert pathlib.Path(entity.path, f"{entity.name}.out").is_symlink()
     assert pathlib.Path(entity.path, f"{entity.name}.err").is_symlink()
+    # Verify symlinks point to the correct run directory
+    expected_out = run_dir / (entity.name + ".out")
+    expected_err = run_dir / (entity.name + ".err")
     assert os.readlink(pathlib.Path(entity.path, f"{entity.name}.out")) == str(
-        status_dir / (entity.name + ".out")
+        expected_out
     )
     assert os.readlink(pathlib.Path(entity.path, f"{entity.name}.err")) == str(
-        status_dir / (entity.name + ".err")
+        expected_err
     )
 
 
 @pytest.mark.parametrize(
-    "entity",
+    "entity_type",
     [
-        pytest.param(ens, id="ensemble"),
-        pytest.param(orc, id="orchestrator"),
-        pytest.param(anon_batch_model, id="model"),
+        pytest.param("ensemble", id="ensemble"),
+        pytest.param("orchestrator", id="orchestrator"),
+        pytest.param("model", id="model"),
     ],
 )
-def test_batch_symlink(entity, test_dir):
+def test_batch_symlink(entity_type, test_dir):
     """Test symlinking historical output files"""
     exp_dir = pathlib.Path(test_dir)
+
+    # Create fresh entities for each test to avoid path conflicts
+    if entity_type == "ensemble":
+        entity = Ensemble(
+            "ens", params={}, run_settings=rs, batch_settings=bs, replicas=3
+        )
+    elif entity_type == "orchestrator":
+        entity = Orchestrator(
+            db_nodes=3, batch=True, launcher="slurm", run_command="srun"
+        )
+    else:  # model
+        batch_model = Model(
+            "batch_test_model",
+            params={},
+            path=test_dir,
+            run_settings=batch_rs,
+            batch_settings=bs,
+        )
+        entity = _AnonymousBatchJob(batch_model)
+
     entity.path = test_dir
-    batch_step, substeps = slurm_controller._create_batch_job_step(entity)
+    # For entities with sub-entities (like Orchestrator), set their paths too
+    if hasattr(entity, "entities"):
+        for sub_entity in entity.entities:
+            sub_entity.path = test_dir
+
+    # Create run_dir to simulate timestamped run structure
+    run_dir = exp_dir / ".smartsim" / "run_test_batch"
+    batch_step, substeps = slurm_controller._create_batch_job_step(entity, run_dir)
 
     # For batch entities, we need to call symlink_output_files correctly
     # Based on how the controller does it, we should pass the individual entities
@@ -148,7 +181,9 @@ def test_symlink_error(test_dir):
         path=pathlib.Path(test_dir, "badpath"),
         run_settings=RunSettings("echo"),
     )
-    bad_step = controller._create_job_step(bad_model)
+    # Create run_dir to avoid using current working directory
+    run_dir = pathlib.Path(test_dir) / ".smartsim" / "run_test_error"
+    bad_step = controller._create_job_step(bad_model, run_dir)
     # The new behavior should auto-create directories and symlinks without errors
     controller.symlink_output_files(bad_step, bad_model)