feat(agent-ui): add embed-frame support for nav items (#1761)

Author: ulivz

multimodal/omni-tars/omni-agent/src/index.ts

@@ -205,11 +205,14 @@ export default class OmniTARSAgent extends ComposableAgent {
           title: 'Code Server',
           link: sandboxBaseUrl + '/code-server/',
           icon: 'code',
+          behavior: 'new-page',
         },
         {
           title: 'VNC',
           link: sandboxBaseUrl + '/vnc/index.html?autoconnect=true',
           icon: 'monitor',
+          behavior: 'embed-frame',
+          autoActive: true,
         },
       ],
     },

multimodal/tarko/agent-ui/docs/runtime-settings-autoactive.md

@@ -0,0 +1,82 @@
+# Runtime Settings-based AutoActive Feature
+
+## Overview
+
+The `autoActive` property in `workspace.navItems` supports dynamic evaluation based on `api/v1/runtime-settings` values, enabling conditional auto-activation of embedded frames.
+
+## How It Works
+
+### autoActive Design Logic
+
+1. **Backward Compatibility**: `autoActive?: boolean` works as before
+2. **String Expressions**: `autoActive?: string` accepts JavaScript expressions
+3. **Dynamic Evaluation**: Navbar fetches runtime settings and evaluates expressions in real-time
+4. **Safe Execution**: Uses `new Function()` with limited scope for security
+
+### Runtime Settings Consumption
+
+1. **API Endpoint**: `apiService.getSessionRuntimeSettings(sessionId)` returns current settings
+2. **Data Structure**: `{ currentValues: Record<string, any> }` contains user settings
+3. **Real-time Updates**: Refetched when session changes
+4. **Expression Context**: Runtime settings passed as `runtimeSettings` parameter
+
+## Usage Examples
+
+### Basic Boolean (Backward Compatible)
+```json
+{
+  "workspace": {
+    "navItems": [
+      {
+        "title": "Code Server",
+        "link": "{prefix}/code-server/",
+        "icon": "code",
+        "behavior": "embed-frame",
+        "autoActive": true
+      }
+    ]
+  }
+}
+```
+
+### Dynamic Expression
+```json
+{
+  "workspace": {
+    "navItems": [
+      {
+        "title": "VNC",
+        "link": "{prefix}/vnc/index.html?autoconnect=true",
+        "icon": "monitor",
+        "behavior": "embed-frame",
+        "autoActive": "runtimeSettings.agentMode === 'game'"
+      }
+    ]
+  }
+}
+```
+
+## Real-world Example
+
+From `examples/webui-config.json`:
+```json
+{
+  "workspace": {
+    "navItems": [
+      {
+        "title": "Code Server",
+        "link": "{prefix}/code-server/",
+        "icon": "code",
+        "behavior": "embed-frame"
+      },
+      {
+        "title": "VNC",
+        "link": "{prefix}/vnc/index.html?autoconnect=true",
+        "icon": "monitor",
+        "behavior": "embed-frame",
+        "autoActive": "debug(runtimeSettings) && runtimeSettings.agentMode === 'game'"
+      }
+    ]
+  }
+}
+```

multimodal/tarko/agent-ui/examples/webui-config.json

@@ -0,0 +1,185 @@
+{
+  "logo": "https://lf3-static.bytednsdoc.com/obj/eden-cn/zyha-aulnh/ljhwZthlaukjlkulzlp/icon.png",
+  "title": "Omni Agent",
+  "subtitle": "Offering seamless integration with a wide range of real-world tools.",
+  "welcomTitle": "Let’s work it out",
+  "welcomePrompts": [],
+  "welcomeCards": [
+    {
+      "title": "2048",
+      "category": "Game",
+      "image": "https://img.poki-cdn.com/cdn-cgi/image/q=78,scq=50,width=80,height=80,fit=cover,f=auto/cb8c967c-4a78-4ffa-8506-cbac69746f4f/2048.png",
+      "agentOptions": {
+        "agentMode": {
+          "id": "game",
+          "link": "https://poki.com/zh/g/2048",
+          "browserMode": "hybrid"
+        }
+      }
+    },
+    {
+      "title": "Four in a Row",
+      "category": "Game",
+      "image": "https://img.poki-cdn.com/cdn-cgi/image/q=78,scq=50,width=80,height=80,fit=cover,f=auto/e80686db-b0fb-4f2c-bd2f-3a89734f102a/four-in-a-row.jpg",
+      "agentOptions": {
+        "agentMode": {
+          "id": "game",
+          "link": "https://poki.com/zh/g/four-in-a-row",
+          "browserMode": "hybrid"
+        }
+      }
+    },
+    {
+      "title": "Block the Pig",
+      "category": "Game",
+      "image": "https://img.poki-cdn.com/cdn-cgi/image/q=78,scq=50,width=80,height=80,fit=cover,f=auto/9fec1234ce2afd5e789f56da463dcffc/block-the-pig.jpeg",
+      "agentOptions": {
+        "agentMode": {
+          "id": "game",
+          "link": "https://poki.com/zh/g/block-the-pig",
+          "browserMode": "hybrid"
+        }
+      }
+    },
+    {
+      "title": "Factory Balls Forever",
+      "category": "Game",
+      "image": "https://img.poki-cdn.com/cdn-cgi/image/q=78,scq=50,width=80,height=80,fit=cover,f=auto/2a503d0a1d9475d6e62c7ea11caa429ab952aa8f500755613a34e66e2196fe82/factory-balls-forever.png",
+      "agentOptions": {
+        "agentMode": {
+          "id": "game",
+          "link": "https://poki.com/zh/g/factory-balls-forever",
+          "browserMode": "hybrid"
+        }
+      }
+    },
+    {
+      "title": "Snake Solver",
+      "category": "Game",
+      "image": "https://img.poki-cdn.com/cdn-cgi/image/q=78,scq=50,width=80,height=80,fit=cover,f=auto/e466baf845544ef47e0dfbd60f127071/snake-solver.png",
+      "agentOptions": {
+        "agentMode": {
+          "id": "game",
+          "link": "https://poki.com/zh/g/snake-solver",
+          "browserMode": "hybrid"
+        }
+      }
+    },
+    {
+      "title": "Penalty Kicks",
+      "category": "Game",
+      "image": "https://img.poki-cdn.com/cdn-cgi/image/q=78,scq=50,width=80,height=80,fit=cover,f=auto/0770daaa8c4ff3c36dd53e6e41f59396/penalty-kicks.png",
+      "agentOptions": {
+        "agentMode": {
+          "id": "game",
+          "link": "https://poki.com/zh/g/penalty-kicks",
+          "browserMode": "hybrid"
+        }
+      }
+    },
+    {
+      "title": "GUI Agent Research",
+      "category": "Research",
+      "prompt": "Search for the latest GUI Agent papers",
+      "image": "https://images.unsplash.com/photo-1551288049-bebda4e38f71?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "UI TARS Information",
+      "category": "Research",
+      "prompt": "Find information about UI TARS",
+      "image": "https://images.unsplash.com/photo-1561070791-2526d30994b5?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "ProductHunt Trends",
+      "category": "Research",
+      "prompt": "Tell me the top 5 most popular projects on ProductHunt today",
+      "image": "https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "Python Hello World",
+      "category": "Code",
+      "prompt": "Write hello world using python",
+      "image": "https://images.unsplash.com/photo-1526379095098-d400fd0bf935?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "Jupyter Math Comparison",
+      "category": "Code",
+      "prompt": "Use jupyter to calculate which is greater in 9.11 and 9.9",
+      "image": "https://images.unsplash.com/photo-1635070041078-e363dbe005cb?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "Reproduce Seed-TARS",
+      "category": "Code",
+      "prompt": "Write code to reproduce seed-tars.com",
+      "image": "https://images.unsplash.com/photo-1627398242454-45a1465c2479?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "Seed-TARS Summary",
+      "category": "Research",
+      "prompt": "Summary seed-tars.com/1.5",
+      "image": "https://images.unsplash.com/photo-1507003211169-0a1dd7228f2d?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "PDF to Markdown Converter",
+      "category": "Code",
+      "prompt": "Write a python code to download the paper https://arxiv.org/abs/2505.12370, and convert the pdf to markdown",
+      "image": "https://images.unsplash.com/photo-1481627834876-b7833e8f5570?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "ByteDance News Website",
+      "category": "Code",
+      "prompt": "Search news about bytedance seed1.6 model, then write a web page in modern style and deploy it",
+      "image": "https://images.unsplash.com/photo-1504711434969-e33886168f5c?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "Transformer Code Sample",
+      "category": "Code",
+      "prompt": "Write a minimal code sample to help me use transformer",
+      "image": "https://images.unsplash.com/photo-1620712943543-bcc4688e7485?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "HuggingFace Dataset Analysis",
+      "category": "Research",
+      "prompt": "Please search for trending datasets on Hugging Face, download the top-ranked dataset, and calculate the total number of characters in the entire dataset.",
+      "image": "https://images.unsplash.com/photo-1551288049-bebda4e38f71?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "Political History Analysis",
+      "category": "Research",
+      "prompt": "Identify the independence process of a twin-island nation where the pro-self-governance political group won thirteen out of seventeen legislative seats in spring 1980 national polls, a second constitutional conference was held at a historic London venue in late 1980, liberation from colonial rule is annually commemorated on November 1st as a public holiday, and an agreement revised the smaller island's local governance legislation for enhanced autonomy. What was the composition of the associated state that preceded its independence?",
+      "image": "https://images.unsplash.com/photo-1529107386315-e1a2ed48a620?w=400&h=300&fit=crop&crop=center"
+    },
+    {
+      "title": "Music Theory Course Builder",
+      "category": "Research",
+      "prompt": "I am a high school music theory teacher and i'm preparing a course on basic music theory to explain knowledge about music names, roll titles, major scales, octave distribution, and physical frequency. Please help me collect enough informations, design fulfilling and authoritative course content with demonstration animations, and finally output them as web page",
+      "image": "https://images.unsplash.com/photo-1493225457124-a3eb161ffa5f?w=400&h=300&fit=crop&crop=center"
+    }
+  ],
+  "workspace": {
+    "navItems": [
+      {
+        "title": "Code Server",
+        "link": "{prefix}/code-server/",
+        "icon": "code",
+        "behavior": "embed-frame"
+      },
+      {
+        "title": "VNC",
+        "link": "{prefix}/vnc/index.html?autoconnect=true",
+        "icon": "monitor",
+        "behavior": "embed-frame",
+        "autoActive": "runtimeSettings.agentMode === 'game'"
+      }
+    ]
+  },
+  "guiAgent": {
+    "defaultScreenshotRenderStrategy": "afterAction",
+    "enableScreenshotRenderStrategySwitch": true,
+    "renderGUIAction": true,
+    "renderBrowserShell": false
+  },
+  "layout": {
+    "enableLayoutSwitchButton": true
+  }
+}

multimodal/tarko/agent-ui/src/common/hooks/useSession.ts

@@ -4,7 +4,13 @@ import { messagesAtom, groupedMessagesAtom } from '../state/atoms/message';
 import { toolResultsAtom } from '../state/atoms/tool';
 
 import { sessionFilesAtom } from '../state/atoms/files';
-import { isProcessingAtom, activePanelContentAtom, connectionStatusAtom } from '../state/atoms/ui';
+import {
+  isProcessingAtom,
+  activePanelContentAtom,
+  connectionStatusAtom,
+  activeEmbedFrameAtom,
+  workspaceDisplayStateAtom,
+} from '../state/atoms/ui';
 import { replayStateAtom } from '../state/atoms/replay';
 import {
   loadSessionsAction,
@@ -35,6 +41,8 @@ export function useSession() {
   const [isProcessing, setIsProcessing] = useAtom(isProcessingAtom);
   const [activePanelContent, setActivePanelContent] = useAtom(activePanelContentAtom);
   const [connectionStatus, setConnectionStatus] = useAtom(connectionStatusAtom);
+  const [activeEmbedFrame, setActiveEmbedFrame] = useAtom(activeEmbedFrameAtom);
+  const [workspaceDisplayState, setWorkspaceDisplayState] = useAtom(workspaceDisplayStateAtom);
 
   const [replayState, setReplayState] = useAtom(replayStateAtom);
 
@@ -91,6 +99,8 @@ export function useSession() {
       isProcessing,
       activePanelContent,
       connectionStatus,
+      activeEmbedFrame,
+      workspaceDisplayState,
 
       replayState,
       sessionMetadata,
@@ -106,6 +116,8 @@ export function useSession() {
       abortQuery,
 
       setActivePanelContent,
+      setActiveEmbedFrame,
+      setWorkspaceDisplayState,
 
       initConnectionMonitoring,
       checkServerStatus,
@@ -122,6 +134,8 @@ export function useSession() {
       isProcessing,
       activePanelContent,
       connectionStatus,
+      activeEmbedFrame,
+      workspaceDisplayState,
       replayState,
       sessionMetadata,
       loadSessions,
@@ -133,6 +147,8 @@ export function useSession() {
       sendMessage,
       abortQuery,
       setActivePanelContent,
+      setActiveEmbedFrame,
+      setWorkspaceDisplayState,
       initConnectionMonitoring,
       checkServerStatus,
       checkSessionStatus,

multimodal/tarko/agent-ui/src/common/state/actions/eventProcessors/handlers/SystemHandler.ts

@@ -5,6 +5,7 @@ import { messagesAtom } from '@/common/state/atoms/message';
 import { sessionPanelContentAtom } from '@/common/state/atoms/ui';
 import { shouldUpdatePanelContent } from '../utils/panelContentUpdater';
 import { ChatCompletionContentPartImage } from '@tarko/agent-interface';
+import { StandardPanelContent } from '@/standalone/workspace/types/index';
 
 export class SystemMessageHandler implements EventHandler<AgentEventStream.SystemEvent> {
   canHandle(event: AgentEventStream.Event): event is AgentEventStream.SystemEvent {
@@ -91,21 +92,20 @@ export class EnvironmentInputHandler
         const currentSessionPanel = currentPanelContent[sessionId];
 
         // Common panel properties
-        const basePanelContent = {
+        const basePanelContent: Partial<StandardPanelContent> = {
           title: event.description || 'Environment Screenshot',
           timestamp: event.timestamp,
-          originalContent: event.content,
           environmentId: event.id,
         };
 
-        let panelContent = null;
+        let panelContent: StandardPanelContent | null = null;
 
         if (isFirstEnvironmentInput) {
           // First environment input: always show as simple image
           panelContent = {
             ...basePanelContent,
             type: 'image',
-            source: imageContent.image_url.url,
+            source: imageContent.image_url.url as string,
           };
         } else if (
           currentSessionPanel?.type === 'browser_vision_control' ||
@@ -115,7 +115,7 @@ export class EnvironmentInputHandler
           panelContent = {
             ...basePanelContent,
             type: 'browser_vision_control',
-            source: null,
+            source: undefined,
             title: event.description || 'Browser Screenshot',
           };
         }