[WIP] Fara-7B in Magentic-UI (#448)

Author: husseinmozannar

README.md

@@ -18,6 +18,9 @@ Magentic-UI is a **research prototype** human-centered AI agent that solves comp
 
 ## ✨ What's New
 
+Microsoft latest agentic model [Fara-7B](https://www.microsoft.com/en-us/research/blog/fara-7b-an-efficient-agentic-model-for-computer-use/) is now integrated in Magentic-UI, read how to launch in <a href="#Magentic-UI with Fara-7B">Fara-7B guide</a>.
+
+
 - **"Tell me When"**: Automate monitoring tasks and repeatable workflows that require web or API access that span minutes to days. *Learn more [here](https://www.microsoft.com/en-us/research/blog/tell-me-when-building-agents-that-can-wait-monitor-and-act/).*
 - **File Upload Support**: Upload any file through the UI for analysis or modification
 - **MCP Agents**: Extend capabilities with your favorite MCP servers
@@ -212,6 +215,59 @@ If you face issues with Docker, please refer to the [TROUBLESHOOTING.md](TROUBLE
 Once the server is running, you can access the UI at <http://localhost:8081>.
 
 
+
+### Magentic-UI with Fara-7B
+
+1) First install magentic-ui with the fara extras:
+
+```bash
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install magentic-ui[fara]
+```
+
+2) In a seperate process, serve the Fara-7B model using vLLM:
+
+```bash
+vllm serve "microsoft/Fara-7B" --port 5000 --dtype auto 
+```
+
+3) First create a `fara_config.yaml` file with the following content:
+
+```yaml
+model_config_local_surfer: &client_surfer
+  provider: OpenAIChatCompletionClient
+  config:
+    model: "microsoft/Fara-7B"
+    base_url: http://localhost:5000/v1
+    api_key: not-needed
+    model_info:
+      vision: true
+      function_calling: true
+      json_output: false
+      family: "unknown" 
+      structured_output: false
+      multiple_system_messages: false
+
+orchestrator_client: *client_surfer
+coder_client: *client_surfer
+web_surfer_client: *client_surfer
+file_surfer_client: *client_surfer
+action_guard_client: *client_surfer
+model_client: *client_surfer
+```
+Note: if you are hosting vLLM on a different port or host, change the `base_url` accordingly.
+
+
+Then launch Magentic-UI with the fara agent:
+
+```bash
+magentic-ui --fara --port 8081 --config fara_config.yaml 
+```
+
+Finally, navigate to <http://localhost:8081> to access the interface!
+
 ### Configuration
 
 #### Model Client Configuration

docker/magentic-ui-browser-docker/build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 IMAGE_NAME=magentic-ui-browser
-IMAGE_VERSION=0.0.1
+IMAGE_VERSION=0.0.2
 REGISTRY=ghcr.io/microsoft
 
 # Check if --push flag is provided or PUSH environment variable is set

docker/magentic-ui-browser-docker/playwright-server.js

@@ -14,7 +14,7 @@ const port = process.env.PLAYWRIGHT_PORT || 37367;
     args: [
       "--start-fullscreen",
       "--start-maximized",
-      "--window-size=1440,1440",
+      "--window-size=1440,900",
       "--window-position=0,0",
       "--disable-infobars",
       "--no-default-browser-check",

docker/magentic-ui-browser-docker/supervisord.conf

@@ -5,7 +5,7 @@ logfile_maxbytes=50MB
 loglevel=info
 
 [program:xvfb]
-command=Xvfb :99 -screen 0 1440x1440x24 -ac -nolisten tcp
+command=Xvfb :99 -screen 0 1440x900x24 -ac -nolisten tcp
 autorestart=true
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -34,7 +34,7 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
 [program:x11vnc]
-command=x11vnc -display :99 -forever -shared -nopw -geometry 1440x1440 -scale 1:1 -nomodtweak
+command=x11vnc -display :99 -forever -shared -nopw -geometry 1440x900 -scale 1:1 -nomodtweak
 autorestart=true
 priority=20
 stdout_logfile=/dev/stdout

docker/magentic-ui-browser-docker/x11-setup.sh

@@ -9,7 +9,7 @@ fi
 xsetroot -solid "#000000"
 
 # Force X11 to use the exact screen dimensions without any offsets
-xrandr --output default --mode 1440x1440 --pos 0x0
+xrandr --output default --mode 1440x900 --pos 0x0
 
 # Set proper DPI settings for the display
 echo "Xft.dpi: 96" | xrdb -merge

fara_config.yaml

@@ -0,0 +1,20 @@
+model_config_local_surfer: &client_surfer
+  provider: OpenAIChatCompletionClient
+  config:
+    model: "microsoft/Fara-7B"
+    base_url: http://localhost:5000/v1
+    api_key: not-needed
+    model_info:
+      vision: true
+      function_calling: true
+      json_output: false
+      family: "unknown" 
+      structured_output: false
+      multiple_system_messages: false
+
+orchestrator_client: *client_surfer
+coder_client: *client_surfer
+web_surfer_client: *client_surfer
+file_surfer_client: *client_surfer
+action_guard_client: *client_surfer
+model_client: *client_surfer

pyproject.toml

@@ -70,6 +70,15 @@ ollama = [
     "autogen-ext[ollama]==0.5.7"
 ]
 
+fara = [
+    "pillow",
+    "tenacity",
+    "pyyaml",
+    "jsonschema",
+    "browserbase",
+    "vllm>=0.10.0",
+]
+
 [project.scripts]
 magentic = "magentic_ui.backend.cli:run"
 magentic-ui = "magentic_ui.backend.cli:run"
@@ -132,7 +141,7 @@ exclude = ["src/magentic_ui/eval"]
 
 [tool.pyright]
 include = ["src", "tests", "samples"]
-exclude = ["src/magentic_ui/eval/benchmarks/assistantbench/evaluate_utils", "src/magentic_ui/agents/web_surfer/_cua_web_surfer.py", "src/magentic_ui/backend/web/routes"]
+exclude = ["src/magentic_ui/eval/benchmarks/assistantbench/evaluate_utils", "src/magentic_ui/agents/web_surfer/_cua_web_surfer.py", "src/magentic_ui/backend/web/routes", "src/magentic_ui/agents/web_surfer/fara", "src/magentic_ui/tools/playwright/playwright_controller_fara.py"]
 typeCheckingMode = "strict"
 reportUnnecessaryIsInstance = false
 reportMissingTypeStubs = false

src/magentic_ui/_docker.py

@@ -9,7 +9,7 @@
 
 DOCKER_REGISTRY = "ghcr.io/microsoft"
 BROWSER_IMAGE = os.getenv(
-    BROWSER_IMAGE_ENV_VAR, f"{DOCKER_REGISTRY}/magentic-ui-browser:0.0.1"
+    BROWSER_IMAGE_ENV_VAR, f"{DOCKER_REGISTRY}/magentic-ui-browser:0.0.2"
 )
 PYTHON_IMAGE = os.getenv(
     PYTHON_IMAGE_ENV_VAR, f"{DOCKER_REGISTRY}/magentic-ui-python-env:0.0.1"

src/magentic_ui/agents/__init__.py

@@ -1,11 +1,12 @@
-from .web_surfer import WebSurfer, WebSurferCUA
+from .web_surfer import WebSurfer, WebSurferCUA, FaraWebSurfer
 from ._coder import CoderAgent
 from ._user_proxy import USER_PROXY_DESCRIPTION
 from .file_surfer import FileSurfer
 
 __all__ = [
     "WebSurfer",
     "WebSurferCUA",
+    "FaraWebSurfer",
     "CoderAgent",
     "USER_PROXY_DESCRIPTION",
     "FileSurfer",

src/magentic_ui/agents/web_surfer/__init__.py

@@ -1,8 +1,10 @@
 from ._web_surfer import WebSurfer, WebSurferConfig
 from ._cua_web_surfer import WebSurferCUA
+from .fara._fara_web_surfer import FaraWebSurfer
 
 __all__ = [
     "WebSurfer",
     "WebSurferConfig",
     "WebSurferCUA",
+    "FaraWebSurfer",
 ]