add dummy user

Author: husseinmozannar

experiments/eval/run.py

@@ -220,7 +220,7 @@ def main() -> None:
         "--simulated-user-type",
         type=str,
         default="none",
-        help="Type of simulated user (co-planning, co-execution, co-planning-and-execution, none)",
+        help="Type of simulated user (co-planning, co-execution, co-planning-and-execution, dummy, none)",
     )
     parser.add_argument(
         "--how-helpful-user-proxy",

experiments/eval/systems/magentic_ui_sim_user_system.py

@@ -23,7 +23,7 @@
 from magentic_ui.types import CheckpointEvent
 from magentic_ui.agents import WebSurfer, CoderAgent, FileSurfer
 from magentic_ui.teams import GroupChat
-from magentic_ui.agents.users import MetadataUserProxy
+from magentic_ui.agents.users import MetadataUserProxy, DummyUserProxy
 from magentic_ui.tools.playwright.browser import VncDockerPlaywrightBrowser
 from magentic_ui.tools.playwright.browser.utils import get_available_port
 from magentic_ui.approval_guard import (
@@ -106,7 +106,11 @@ def __init__(
         self,
         name: str = "MagenticUISimUserSystem",
         simulated_user_type: Literal[
-            "co-planning", "co-execution", "co-planning-and-execution", "none"
+            "co-planning",
+            "co-execution",
+            "co-planning-and-execution",
+            "none",
+            "dummy",
         ] = "none",
         how_helpful_user_proxy: Literal["strict", "soft", "no_hints"] = "soft",
         web_surfer_only: bool = False,
@@ -217,7 +221,7 @@ async def _runner() -> Tuple[str, List[str]]:
                 if self.simulated_user_type in ["co-execution", "none"]
                 else True,
                 autonomous_execution=True
-                if self.simulated_user_type in ["co-planning", "none"]
+                if self.simulated_user_type in ["co-planning", "none", "dummy"]
                 else False,
                 allow_follow_up_input=False,
                 final_answer_prompt=FINAL_ANSWER_PROMPT,
@@ -312,6 +316,10 @@ def get_model_client(
 
             if self.simulated_user_type == "none":
                 user_proxy = None
+            elif self.simulated_user_type == "dummy":
+                user_proxy = DummyUserProxy(
+                    name="user_proxy",
+                )
             else:
                 user_proxy = MetadataUserProxy(
                     name="user_proxy",
@@ -346,7 +354,7 @@ def get_model_client(
                 from autogen_core import CancellationToken
                 from autogen_core.models import UserMessage
 
-                prompt = f"""Rewrite the following helpful hints to help solve the task, but remove any information that directly reveals the answer. \nKeep the hints as close to the original as possible but remove any information that directly reveals the answer.\nHelpful hints: {task_metadata}\n\nAnswer: {getattr(task, 'ground_truth', '')}\n\nDo not include anything else in your response except the rewritten hints.\nRewritten helpful hints:"""
+                prompt = f"""Rewrite the following helpful hints to help solve the task, but remove any information that directly reveals the answer. \nKeep the hints as close to the original as possible but remove any information that directly reveals the answer.\nHelpful hints: {task_metadata}\n\nAnswer: {getattr(task, "ground_truth", "")}\n\nDo not include anything else in your response except the rewritten hints.\nRewritten helpful hints:"""
                 result = await model_client_orch.create(
                     messages=[UserMessage(content=prompt, source="user")],
                     cancellation_token=CancellationToken(),
@@ -418,9 +426,9 @@ def get_model_client(
                     # remove the "FINAL ANSWER:" part and get the string after it
                     answer = answer.split("FINAL ANSWER:")[1].strip()
 
-            assert isinstance(
-                answer, str
-            ), f"Expected answer to be a string, got {type(answer)}"
+            assert isinstance(answer, str), (
+                f"Expected answer to be a string, got {type(answer)}"
+            )
 
             # save the usage of each of the client in a usage json file
             def get_usage(model_client: ChatCompletionClient) -> Dict[str, int]: