feat: SSE tools search improvements, engine updates, skill refinements and functional tests

Author: razvan

internal/service/engine/engine.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -359,6 +360,13 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 
 	all := append(primaryResults, otherResults...)
 
+	// Global sort by score descending across all language results, then cap to limit.
+	// Without this, primary-language results always win regardless of score.
+	sort.Slice(all, func(i, j int) bool { return all[i].Score > all[j].Score })
+	if limit > 0 && len(all) > limit {
+		all = all[:limit]
+	}
+
 	// If nothing was found and there were errors, surface the error
 	if len(all) == 0 && firstErr != nil {
 		return nil, fmt.Errorf("search failed: %w", firstErr)

internal/service/tools/search_local_index.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -249,18 +250,14 @@ func (t *SearchLocalIndexTool) Execute(ctx context.Context, params map[string]in
 				wg.Add(1)
 				go func(name string) {
 					defer wg.Done()
-					// ExactSearch first — zero embedding, deterministic
+					// ExactSearch only — zero embedding, deterministic.
+					// No fallback to embedding search: relation names are often stdlib/external
+					// symbols not in the local index, and each embedding call costs ~N seconds.
 					res, err := t.engine.SearchByName(ctx, wsID, name, 2)
 					if err != nil || len(res) == 0 {
-						// Fallback: embedding search (e.g. external/stdlib symbols not in index)
-						subRes, sErr := t.engine.SearchCode(ctx, filePath, name, 2, false)
-						if sErr == nil && subRes != nil {
-							res = subRes.Results
-						}
-					}
-					if len(res) > 0 {
-						subChan <- subResult{targetName: name, results: res}
+						return
 					}
+					subChan <- subResult{targetName: name, results: res}
 				}(targetName)
 			}
 
@@ -294,6 +291,14 @@ func (t *SearchLocalIndexTool) Execute(ctx context.Context, params map[string]in
 		}
 	}
 
+	// Sort all descriptors (primary + graph expansions) by score descending
+	// so the highest-relevance results are always surfaced first in the response.
+	sort.Slice(descriptors, func(i, j int) bool {
+		si, _ := descriptors[i]["score"].(float32)
+		sj, _ := descriptors[j]["score"].(float32)
+		return si > sj
+	})
+
 	response.Context.Telemetry = telemetry.CalculateSavings(baselineBytes, actualBytes)
 	response.Data = descriptors
 	return response.JSON()

internal/service/tools/tests/search_test.go

@@ -159,24 +159,18 @@ var _ = Describe("RagSearchCodeTool", func() {
 					"SearchCodeOnly with expansion limit should not be called when ExactSearch succeeds")
 			})
 
-			It("should fall back to embedding when ExactSearch finds nothing for a dependency", func() {
-				expansionFallbackCalls := 0
+			It("should NOT fall back to embedding when ExactSearch finds nothing for a dependency", func() {
+				// Regression test: graph expansion must NOT trigger embedding calls for
+				// relations not found in the local index (stdlib/external symbols).
+				// Each fallback embedding call costs ~N seconds serialized through Ollama.
+				embeddingCallsForExpansion := 0
 
 				mockStore.SearchCodeOnlyFunc = func(ctx context.Context, col string, q storage.SearchQuery) ([]storage.SearchResult, error) {
 					if q.Limit == 2 {
-						expansionFallbackCalls++
-						// Fallback expansion search — return the dependency
-						return []storage.SearchResult{
-							{
-								Score: 0.7,
-								Point: storage.Point{
-									ID:      "ext-helper-id",
-									Payload: map[string]interface{}{"name": "ExternalHelper"},
-								},
-							},
-						}, nil
+						// This would be a fallback expansion embedding — must NOT be called.
+						embeddingCallsForExpansion++
 					}
-					// Main query
+					// Main query result — has a relation to an external/stdlib symbol
 					return []storage.SearchResult{
 						{
 							Score: 1.0,
@@ -193,7 +187,7 @@ var _ = Describe("RagSearchCodeTool", func() {
 					}, nil
 				}
 
-				// ExactSearch finds nothing — triggers fallback
+				// ExactSearch finds nothing — external symbol not in index
 				mockStore.ExactSearchFunc = func(ctx context.Context, col string, filters map[string]interface{}, limit int) ([]storage.SearchResult, error) {
 					return []storage.SearchResult{}, nil
 				}
@@ -204,14 +198,14 @@ var _ = Describe("RagSearchCodeTool", func() {
 				var resp tools.ToolResponse
 				json.Unmarshal([]byte(resJSON), &resp)
 				Expect(resp.Status).To(Equal("success"), "Error: "+resp.Error)
-				Expect(resp.Message).To(ContainSubstring("Auto-fetched 1 related dependencies"))
 
+				// Only root result — dependency was skipped (not in index, no fallback)
 				data := resp.Data.([]interface{})
-				Expect(data).To(HaveLen(2))
+				Expect(data).To(HaveLen(1))
 
-				// Fallback was triggered: SearchCodeOnly called with limit=2 at least once
-				Expect(expansionFallbackCalls).To(BeNumerically(">=", 1),
-					"Fallback embedding search should be called when ExactSearch returns no results")
+				// No embedding fallback — critical for performance
+				Expect(embeddingCallsForExpansion).To(Equal(0),
+					"Embedding fallback must NOT be called for graph expansion: external/stdlib symbols not in index would cause N×26s latency")
 			})
 
 			It("should deduplicate expansion targets when same dependency appears multiple times in relations", func() {

internal/skills/embedded/ragcode-sse/SKILL.md

@@ -12,24 +12,69 @@ This skill shows any agent how to call RagCode MCP directly via HTTP + Server‑
 ## 🔌 Endpoints
 
 ```
-GET  /sse        # open the event stream (keep-alive)
-POST /messages   # send JSON-RPC requests
+GET  /sse                        # Opens the persistent stream
+POST /sse?sessionid=ID           # Send JSON-RPC (exact URL from the endpoint event)
 ```
 
-Set the port with the `-http-port` flag (default `3000`). Example URLs:
+### 🔑 Session Handshake
+1. Connect to `GET /sse` and keep the connection open.
+2. The first SSE event is `event: endpoint`.
+3. Its data is the full POST URL, e.g.: `data: /sse?sessionid=H3QCVDP32TP3RBZQ5WLMEJSRF`
+4. **Required**: send `initialize` + `notifications/initialized` before any `tools/call`.
+5. All responses are delivered back on the open SSE stream.
 
-```
-SSE stream  : http://localhost:3000/sse
-Send message: http://localhost:3000/messages
+### 🧾 Bash Script (Tested and working)
+```bash
+#!/bin/bash
+SSE_FILE=$(mktemp)
+curl -s -N -H 'Accept: text/event-stream' http://localhost:3000/sse >> "$SSE_FILE" &
+SSE_PID=$!
+sleep 1
+
+# Extract the POST URL from the endpoint event
+ENDPOINT=$(grep 'data:' "$SSE_FILE" | head -1 | sed 's/^data: //' | tr -d '[:space:]')
+POST_URL="http://localhost:3000${ENDPOINT}"
+
+# 1. MCP handshake - initialize
+curl -s -X POST "$POST_URL" -H 'Content-Type: application/json' -d '{
+  "jsonrpc": "2.0", "id": 1, "method": "initialize",
+  "params": {"protocolVersion": "2024-11-05", "capabilities": {},
+              "clientInfo": {"name": "my-agent", "version": "1.0"}}
+}' && sleep 1
+
+# 2. MCP handshake - initialized notification
+curl -s -X POST "$POST_URL" -H 'Content-Type: application/json' -d '{
+  "jsonrpc": "2.0", "method": "notifications/initialized"
+}' && sleep 1
+
+# 3. Tool call
+curl -s -X POST "$POST_URL" -H 'Content-Type: application/json' -d '{
+  "jsonrpc": "2.0", "id": 2, "method": "tools/call",
+  "params": {
+    "name": "rag_search_code",
+    "arguments": {
+      "query": "automatic update implementation",
+      "file_path": "/your/project/file.go"
+    }
+  }
+}'
+
+# 4. Read the response from the SSE stream
+sleep 12
+kill $SSE_PID 2>/dev/null
+cat "$SSE_FILE"
+rm "$SSE_FILE"
 ```
 
 ---
 
 ## 🧠 Protocol Basics
 
-1. Keep a persistent SSE connection to `/sse` (subscribe to events).
-2. Send JSON-RPC payloads to `/messages`.
-3. Responses arrive asynchronously on the SSE stream (matching `id`).
+1. Keep a persistent SSE connection via `GET /sse`.
+2. Read the first `event: endpoint` to get the POST URL (`/sse?sessionid=ID`).
+3. Send `initialize` + `notifications/initialized` (MCP handshake) before any tool call.
+4. Send JSON-RPC payloads via `POST /sse?sessionid=ID`.
+5. Responses arrive asynchronously on the SSE stream (match by `id`).
 
 ### JSON-RPC Template
 
@@ -55,16 +100,32 @@ Use other MCP methods such as `tools/list`, `ping`, etc.
 
 ## 🧾 curl Quick Start
 
-Open a stream (terminal tab A):
+Tab A — open the stream and note the POST URL:
 
 ```bash
-curl -N http://localhost:3000/sse
+curl -N -H 'Accept: text/event-stream' http://localhost:3000/sse
+# First output: data: /sse?sessionid=XXXXXX  <-- this is your POST_URL
 ```
 
-Send a message (terminal tab B):
+Tab B — full sequence (replace `XXXXXX` with the real sessionid):
 
 ```bash
-curl -X POST http://localhost:3000/messages \
+POST_URL="http://localhost:3000/sse?sessionid=XXXXXX"
+
+# Step 1: initialize
+curl -X POST "$POST_URL" -H 'Content-Type: application/json' -d '{
+  "jsonrpc": "2.0", "id": 1, "method": "initialize",
+  "params": {"protocolVersion": "2024-11-05", "capabilities": {},
+              "clientInfo": {"name": "curl-client", "version": "1.0"}}
+}' && sleep 1
+
+# Step 2: notifications/initialized
+curl -X POST "$POST_URL" -H 'Content-Type: application/json' -d '{
+  "jsonrpc": "2.0", "method": "notifications/initialized"
+}' && sleep 1
+
+# Step 3: tools/call
+curl -X POST "$POST_URL" \
      -H 'Content-Type: application/json' \
      -d '{
   "jsonrpc": "2.0",
@@ -87,25 +148,42 @@ Watch the SSE tab for the response.
 ## 🐍 Python Example
 
 ```python
-import json, requests, sseclient
+import json, threading, requests, sseclient
 
-SSE_URL = "http://localhost:3000/sse"
-MSG_URL = "http://localhost:3000/messages"
+BASE_URL = "http://localhost:3000"
 
-payload = {
-    "jsonrpc": "2.0",
-    "id": "list-tools",
-    "method": "tools/list",
-    "params": {}
-}
+# 1. Open SSE stream and extract POST URL
+resp = requests.get(f"{BASE_URL}/sse", stream=True, headers={"Accept": "text/event-stream"})
+client = sseclient.SSEClient(resp)
 
-requests.post(MSG_URL, json=payload, timeout=10)
-client = sseclient.SSEClient(SSE_URL)
+post_url = None
+for event in client.events():
+    if event.event == "endpoint":
+        post_url = BASE_URL + event.data.strip()
+        break
+
+def post(payload):
+    requests.post(post_url, json=payload, timeout=10)
+
+# 2. MCP Handshake
+post({"jsonrpc": "2.0", "id": 1, "method": "initialize",
+      "params": {"protocolVersion": "2024-11-05", "capabilities": {},
+                 "clientInfo": {"name": "py-agent", "version": "1.0"}}})
+post({"jsonrpc": "2.0", "method": "notifications/initialized"})
+
+# 3. Tool call
+post({"jsonrpc": "2.0", "id": 2, "method": "tools/call",
+      "params": {"name": "rag_search_code",
+                 "arguments": {"query": "workspace registry",
+                               "file_path": "/your/project/main.py"}}})
+
+# 4. Read responses
 for event in client.events():
-    print(event.data)
+    if event.data:
+        print(json.loads(event.data))
 ```
 
-Any SSE client works; just keep reading events and match IDs.
+Any SSE client works; keep the stream open and match responses by `id`.
 
 ---
 
@@ -128,17 +206,19 @@ Any SSE client works; just keep reading events and match IDs.
 
 | Symptom | Fix |
 | --- | --- |
-| No response | Ensure SSE connection is open; use unique `id`s per request. |
-| 4xx on `/messages` | Check JSON validity and `Content-Type: application/json`. |
-| Workspace errors | Always pass `arguments.file_path` for context detection. |
+| No response | Ensure SSE is open and MCP handshake was done; use unique `id` per request. |
+| 4xx on POST | Check JSON and `Content-Type: application/json`; POST goes to `/sse?sessionid=ID`, not `/messages`. |
+| `sessionid must be provided` | Extract sessionid using `sed 's/^data: //'` on the `data:` line from the SSE stream. |
+| Workspace errors | Always pass an absolute path in `arguments.file_path` for workspace detection. |
 
 ---
 
 ## ✅ Summary
 
-- **Endpoints**: `GET /sse`, `POST /messages`.
+- **Endpoints**: `GET /sse` (stream), `POST /sse?sessionid=ID` (messages).
+- **Required handshake**: `initialize` → `notifications/initialized` → `tools/call`.
 - **Protocol**: JSON-RPC 2.0 (MCP spec).
-- **Examples**: Provided for curl and Python.
-- **Tool discovery**: `list_tools` response.
+- **Examples**: Provided for curl (bash) and Python.
+- **Tool discovery**: call `tools/list` after handshake.
 
 Install this skill to teach any agent how to drive RagCode MCP over SSE immediately.