perf: reduce indexer embed concurrency and add progress display

- pkg/indexer/service.go: numWorkers=1 (Ollama is serial, parallelism
  only queues requests and blocks search queries); add \r progress line
  showing % files remaining during indexing
- pkg/llm/ollama.go: remove unused mu sync.Mutex field
- internal/service/search/search.go: CollectionExists cache (60s TTL)
- internal/service/tools/search_local_index.go: remove semaphore from
  graph expansion (ExactSearch is metadata-only, no Ollama calls)
- internal/service/engine/engine.go: TIMER instrumentation for SearchCode

Author: razvan

internal/service/engine/engine.go

@@ -257,10 +257,13 @@ func (e *ErrIndexingStarted) Error() string {
 // embeds the query ONCE, then fans out in parallel to all language collections.
 // includeDocs=false searches code only. Triggers background indexing if needed.
 func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, limit int, includeDocs bool) (*SearchCodeResult, error) {
+	t0 := time.Now()
+
 	wctx, err := e.DetectContext(ctx, filePath)
 	if err != nil {
 		return nil, err
 	}
+	log.Printf("[TIMER] SearchCode detect_context=%v", time.Since(t0))
 
 	// Primary language from file extension
 	primaryLang := "go"
@@ -270,7 +273,9 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 
 	// Ensure at least the primary collection exists before embedding (fast fail + indexing trigger)
 	primaryColl := wctx.CollectionName(primaryLang)
+	t1 := time.Now()
 	exists, err := e.search.CollectionExists(ctx, primaryColl)
+	log.Printf("[TIMER] SearchCode collection_exists_primary=%v (cached=%v)", time.Since(t1), time.Since(t1) < time.Millisecond)
 	if err != nil {
 		return nil, fmt.Errorf("failed to check collection: %w", err)
 	}
@@ -293,7 +298,9 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 		langs = []string{"go", "python", "php", "html"}
 	}
 
+	t2 := time.Now()
 	vector, err := e.search.EmbedQuery(ctx, queryText)
+	log.Printf("[TIMER] SearchCode embed=%v", time.Since(t2))
 	if err != nil {
 		return nil, fmt.Errorf("embedding failed: %w", err)
 	}
@@ -303,16 +310,19 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 		coll    string
 		results []storage.SearchResult
 		err     error
+		elapsed time.Duration
 	}
 
 	resultsChan := make(chan langResult, len(langs))
 	var wg sync.WaitGroup
+	t3 := time.Now()
 
 	for _, lang := range langs {
 		coll := wctx.CollectionName(lang)
 		wg.Add(1)
 		go func(l, c string) {
 			defer wg.Done()
+			gt := time.Now()
 			ok, chkErr := e.search.CollectionExists(ctx, c)
 			if chkErr != nil || !ok {
 				return
@@ -324,19 +334,21 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 			} else {
 				res, sErr = e.search.SearchCodeWithVector(ctx, c, vector, limit)
 			}
+			elapsed := time.Since(gt)
 			if sErr != nil {
 				log.Printf("[WARN] SearchCode: fan-out failed for %s: %v", c, sErr)
-				resultsChan <- langResult{lang: l, coll: c, err: sErr}
+				resultsChan <- langResult{lang: l, coll: c, err: sErr, elapsed: elapsed}
 				return
 			}
 			if len(res) > 0 {
-				resultsChan <- langResult{lang: l, coll: c, results: res}
+				resultsChan <- langResult{lang: l, coll: c, results: res, elapsed: elapsed}
 			}
 		}(lang, coll)
 	}
 
 	wg.Wait()
 	close(resultsChan)
+	log.Printf("[TIMER] SearchCode fanout_total=%v (langs=%d)", time.Since(t3), len(langs))
 
 	// Merge: primary lang results first, others appended; surface first error if no results
 	var primaryResults []storage.SearchResult
@@ -347,11 +359,13 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 
 	for lr := range resultsChan {
 		if lr.err != nil {
+			log.Printf("[TIMER] SearchCode lang=%s err elapsed=%v", lr.lang, lr.elapsed)
 			if firstErr == nil {
 				firstErr = lr.err
 			}
 			continue
 		}
+		log.Printf("[TIMER] SearchCode lang=%s hits=%d elapsed=%v", lr.lang, len(lr.results), lr.elapsed)
 		if lr.coll == primaryColl {
 			primaryResults = lr.results
 		} else {
@@ -373,6 +387,9 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 		return nil, fmt.Errorf("search failed: %w", firstErr)
 	}
 
+	log.Printf("[TIMER] SearchCode TOTAL=%v (detect=%v embed=%v fanout=%v)",
+		time.Since(t0), t1.Sub(t0), t2.Sub(t1), time.Since(t3))
+
 	return &SearchCodeResult{
 		Results:         all,
 		WorkspaceRoot:   wctx.Root,

internal/service/search/search.go

@@ -7,15 +7,25 @@ import (
 	"math"
 	"sort"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/doITmagic/rag-code-mcp/internal/service/internalutil"
 	"github.com/doITmagic/rag-code-mcp/pkg/llm"
 	"github.com/doITmagic/rag-code-mcp/pkg/storage"
 )
 
+const collectionExistsTTL = 60 * time.Second
+
+type collectionCacheEntry struct {
+	exists bool
+	expiry time.Time
+}
+
 type Service struct {
-	embedder llm.Provider
-	store    storage.VectorStore
+	embedder        llm.Provider
+	store           storage.VectorStore
+	collectionCache sync.Map // map[string]*collectionCacheEntry
 }
 
 // NewService creates a new search service.
@@ -101,8 +111,33 @@ func (s *Service) ExactSearch(ctx context.Context, collection string, filter map
 }
 
 // CollectionExists checks whether a collection exists in the vector store.
+// Results are cached for 60 s to avoid hammering Qdrant on every fan-out query.
 func (s *Service) CollectionExists(ctx context.Context, collection string) (bool, error) {
-	return s.store.CollectionExists(ctx, collection)
+	now := time.Now()
+	if v, ok := s.collectionCache.Load(collection); ok {
+		entry := v.(*collectionCacheEntry)
+		if now.Before(entry.expiry) {
+			return entry.exists, nil
+		}
+	}
+	exists, err := s.store.CollectionExists(ctx, collection)
+	if err != nil {
+		return false, err
+	}
+	// Cache only positive results — "doesn't exist" can flip quickly after indexing.
+	if exists {
+		s.collectionCache.Store(collection, &collectionCacheEntry{
+			exists: true,
+			expiry: now.Add(collectionExistsTTL),
+		})
+	}
+	return exists, nil
+}
+
+// InvalidateCollectionCache removes the cached existence entry for a collection.
+// Call this after creating or dropping a collection.
+func (s *Service) InvalidateCollectionCache(collection string) {
+	s.collectionCache.Delete(collection)
 }
 
 // HybridSearch combines semantic search with basic lexical re-ranking.

internal/service/tools/search_local_index.go

@@ -231,7 +231,6 @@ func (t *SearchLocalIndexTool) Execute(ctx context.Context, params map[string]in
 			var wg sync.WaitGroup
 			seenTargets := make(map[string]bool)
 			expanded := 0
-			sem := make(chan struct{}, 3) // limit concurrent DB lookups
 
 			for _, relRaw := range relList {
 				if expanded >= maxExpansions {
@@ -251,8 +250,6 @@ func (t *SearchLocalIndexTool) Execute(ctx context.Context, params map[string]in
 				wg.Add(1)
 				go func(name string) {
 					defer wg.Done()
-					sem <- struct{}{}
-					defer func() { <-sem }()
 					// ExactSearch only — zero embedding, deterministic.
 					// No fallback to embedding search: relation names are often stdlib/external
 					// symbols not in the local index, and each embedding call costs ~N seconds.

pkg/indexer/service.go

@@ -11,6 +11,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/doITmagic/rag-code-mcp/pkg/llm"
 	"github.com/doITmagic/rag-code-mcp/pkg/parser"
@@ -129,7 +130,8 @@ func (s *Service) IndexWorkspace(ctx context.Context, root string, collection st
 		}
 	}
 
-	log.Printf("[INFO] Indexing %d changed files in %s (Language: %s)", len(changedFiles), root, opts.Language)
+	totalFiles := len(changedFiles)
+	log.Printf("[INFO] Indexing %d changed files in %s (Language: %s)", totalFiles, root, opts.Language)
 
 	// 4. Process changed files in parallel (file-level worker pool).
 	// State.UpdateFile is mutex-protected; errors are collected safely.
@@ -141,23 +143,28 @@ func (s *Service) IndexWorkspace(ctx context.Context, root string, collection st
 		numFileWorkers = 8
 	}
 
-	filePaths := make(chan string, len(changedFiles))
+	filePaths := make(chan string, totalFiles)
 	for _, p := range changedFiles {
 		filePaths <- p
 	}
 	close(filePaths)
 
 	var (
-		fileWg   sync.WaitGroup
-		errMu    sync.Mutex
-		fileErrs []string
+		fileWg    sync.WaitGroup
+		errMu     sync.Mutex
+		fileErrs  []string
+		doneFiles atomic.Int64
 	)
 
 	for i := 0; i < numFileWorkers; i++ {
 		fileWg.Add(1)
 		go func() {
 			defer fileWg.Done()
 			for path := range filePaths {
+				n := int(doneFiles.Add(1))
+				remaining := totalFiles - n
+				pct := remaining * 100 / totalFiles
+				fmt.Fprintf(os.Stderr, "\r[INDEX] %s: %d%% (%d/%d files left)   ", opts.Language, pct, remaining, totalFiles)
 				if err := s.IndexFile(ctx, collection, path, state); err != nil {
 					log.Printf("[ERROR] Failed to index %s: %v", path, err)
 					errMu.Lock()
@@ -169,6 +176,7 @@ func (s *Service) IndexWorkspace(ctx context.Context, root string, collection st
 	}
 
 	fileWg.Wait()
+	fmt.Fprintf(os.Stderr, "\r[INDEX] %s: done (%d files indexed)            \n", opts.Language, totalFiles)
 
 	if len(fileErrs) > 0 {
 		log.Printf("[WARN] %d file(s) failed to index in %s", len(fileErrs), root)
@@ -224,14 +232,9 @@ func (s *Service) IndexItems(ctx context.Context, collection string, symbols []p
 		return nil
 	}
 
-	// Dynamic worker pool based on CPU
-	numWorkers := 4
-	if n := runtime.NumCPU() / 2; n > numWorkers {
-		numWorkers = n
-	}
-	if numWorkers > 16 {
-		numWorkers = 16 // Cap at 16 to avoid overwhelming downstream services
-	}
+	// Ollama processes embeds serially — multiple workers only queue up requests
+	// and increase latency for concurrent search queries. Keep 1 worker.
+	numWorkers := 1
 
 	type result struct {
 		point storage.Point
@@ -308,6 +311,7 @@ func (s *Service) IndexItems(ctx context.Context, collection string, symbols []p
 		}
 	}
 
+	log.Printf("[INFO] Indexed %d symbols into %s", len(allPoints), collection)
 	return nil
 }
 

pkg/llm/ollama.go

@@ -16,8 +16,8 @@ import (
 type OllamaLLMProvider struct {
 	embedModel llms.Model
 	embedName  string
-	cachedDim  uint64
-	dimOnce    sync.Once
+	cachedDim uint64
+	dimOnce   sync.Once
 }
 
 // NewOllamaLLMProvider creates a new Ollama provider configured for embedding only.